01HQP784HCMEM70Y8VXCFJQGKM: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HQP78EAFPPM4CQD1WZTE3QSM

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-27T21:40:23.331Zjob dependencies complete; ready to run (waiting for 16 m 6 s)
22024-02-27T21:41:32.423Zjob assigned to worker 01HQP8655212FN3V109023NK21 (queued for 1 m 9 s)
32024-02-27T21:41:32.988Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-27T21:42:08.018Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-27T21:42:08.018Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-27T21:42:30.968Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-27T21:42:30.968Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-27T21:43:02.122Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-27T21:43:02.123Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-27T21:43:16.923Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-27T21:43:16.930Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-27T21:43:16.962Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-27T21:43:16.962Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-27T21:43:16.990Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-27T21:43:16.990Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-27T21:43:17.015Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-27T21:43:17.016Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-27T21:43:17.053Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-27T21:43:17.054Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-27T21:43:17.601Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-27T21:43:17.601Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-27T21:43:17.628Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-27T21:43:17.629Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-27T21:43:17.652Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-27T21:43:17.662Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-27T21:43:41.632Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-27T21:43:41.632Zstarting task 0: "setup"
282024-02-27T21:43:41.690Z++ uname -s
292024-02-27T21:43:41.704Z+ kern=SunOS
302024-02-27T21:43:41.705Z+ case "$kern" in
312024-02-27T21:43:41.705Z+ groupadd -g 12345 build
322024-02-27T21:43:41.705Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-27T21:43:43.705Z+ zfs create -o mountpoint=/work rpool/work
342024-02-27T21:43:43.979Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-27T21:43:44.013Z+ home_fs=zfs
362024-02-27T21:43:44.013Z+ [[ zfs == autofs ]]
372024-02-27T21:43:44.013Z+ mkdir -p /home/build
382024-02-27T21:43:44.013Z+ chown build:build /home/build /work
392024-02-27T21:43:45.969Z+ chmod 0700 /home/build /work
402024-02-27T21:43:45.980Zprocess exited: duration 4399 ms, exit code 0
 
412024-02-27T21:43:46.006Zstarting task 1: "authentication"
422024-02-27T21:43:46.026Zprocess exited: duration 19 ms, exit code 0
 
432024-02-27T21:43:46.071Zstarting task 2: "build"
442024-02-27T21:43:46.084Z+ banner cores
452024-02-27T21:43:46.114Z
462024-02-27T21:43:46.114Z #### #### ##### ###### ####
472024-02-27T21:43:46.114Z # # # # # # # #
482024-02-27T21:43:46.114Z # # # # # ##### ####
492024-02-27T21:43:46.115Z # # # ##### # #
502024-02-27T21:43:46.115Z # # # # # # # # #
512024-02-27T21:43:46.115Z #### #### # # ###### ####
522024-02-27T21:43:46.116Z
532024-02-27T21:43:46.137Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-27T21:43:46.137Z+ echo 'input bins dir contains:'
552024-02-27T21:43:46.137Zinput bins dir contains:
562024-02-27T21:43:46.137Z+ ls -ltr /input/build/work/bins
572024-02-27T21:43:46.138Ztotal 740604
582024-02-27T21:43:46.138Z-rw-r--r-- 1 root root 109635226 Feb 27 21:42 crucible-downstairs.gz
592024-02-27T21:43:46.138Z-rw-r--r-- 1 root root 91526609 Feb 27 21:42 crucible-hammer.gz
602024-02-27T21:43:46.138Z-rw-r--r-- 1 root root 105659381 Feb 27 21:43 crutest.gz
612024-02-27T21:43:46.138Z-rw-r--r-- 1 root root 72121864 Feb 27 21:43 dsc.gz
622024-02-27T21:43:46.138Z+ echo 'input script dir contains:'
632024-02-27T21:43:46.138Zinput script dir contains:
642024-02-27T21:43:46.138Z+ ls -ltr /input/build/work/scripts
652024-02-27T21:43:46.138Ztotal 39
662024-02-27T21:43:46.138Z-rw-r--r-- 1 root root 3056 Feb 27 21:43 perf-downstairs-tick.d
672024-02-27T21:43:46.139Z-rw-r--r-- 1 root root 1738 Feb 27 21:43 test_ds.sh
682024-02-27T21:43:46.139Z-rw-r--r-- 1 root root 3746 Feb 27 21:43 test_live_repair.sh
692024-02-27T21:43:46.139Z-rw-r--r-- 1 root root 6416 Feb 27 21:43 test_repair.sh
702024-02-27T21:43:46.139Z-rw-r--r-- 1 root root 2969 Feb 27 21:43 test_replay.sh
712024-02-27T21:43:46.139Z-rw-r--r-- 1 root root 9465 Feb 27 21:43 test_up.sh
722024-02-27T21:43:46.139Z-rw-r--r-- 1 root root 2824 Feb 27 21:43 upstairs_info.d
732024-02-27T21:43:46.139Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-27T21:43:46.139Z+ echo ' chmod input script dir contains:'
752024-02-27T21:43:46.139Z chmod input script dir contains:
762024-02-27T21:43:46.139Z+ ls -ltr /input/build/work/scripts
772024-02-27T21:43:46.139Ztotal 39
782024-02-27T21:43:46.139Z-rwxr-xr-x 1 root root 3056 Feb 27 21:43 perf-downstairs-tick.d
792024-02-27T21:43:46.139Z-rwxr-xr-x 1 root root 1738 Feb 27 21:43 test_ds.sh
802024-02-27T21:43:46.140Z-rwxr-xr-x 1 root root 3746 Feb 27 21:43 test_live_repair.sh
812024-02-27T21:43:46.140Z-rwxr-xr-x 1 root root 6416 Feb 27 21:43 test_repair.sh
822024-02-27T21:43:46.140Z-rwxr-xr-x 1 root root 2969 Feb 27 21:43 test_replay.sh
832024-02-27T21:43:46.140Z-rwxr-xr-x 1 root root 9465 Feb 27 21:43 test_up.sh
842024-02-27T21:43:46.140Z-rwxr-xr-x 1 root root 2824 Feb 27 21:43 upstairs_info.d
852024-02-27T21:43:46.140Z+ banner unpack
862024-02-27T21:43:46.140Z
872024-02-27T21:43:46.140Z # # # # ##### ## #### # #
882024-02-27T21:43:46.140Z # # ## # # # # # # # # #
892024-02-27T21:43:46.140Z # # # # # # # # # # ####
902024-02-27T21:43:46.140Z # # # # # ##### ###### # # #
912024-02-27T21:43:46.140Z # # # ## # # # # # # #
922024-02-27T21:43:46.141Z #### # # # # # #### # #
932024-02-27T21:43:46.141Z
942024-02-27T21:43:46.141Z+ mkdir -p /var/tmp/bins
952024-02-27T21:43:46.141Z+ for t in "$input/bins/"*.gz
962024-02-27T21:43:46.141Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-27T21:43:46.157Z+ b=crucible-downstairs.gz
982024-02-27T21:43:46.157Z+ b=crucible-downstairs
992024-02-27T21:43:46.157Z+ gunzip
1002024-02-27T21:43:48.704Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-27T21:43:48.718Z+ for t in "$input/bins/"*.gz
1022024-02-27T21:43:48.718Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-27T21:43:48.718Z+ b=crucible-hammer.gz
1042024-02-27T21:43:48.719Z+ b=crucible-hammer
1052024-02-27T21:43:48.719Z+ gunzip
1062024-02-27T21:43:50.803Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-27T21:43:50.827Z+ for t in "$input/bins/"*.gz
1082024-02-27T21:43:50.827Z++ basename /input/build/work/bins/crutest.gz
1092024-02-27T21:43:50.828Z+ b=crutest.gz
1102024-02-27T21:43:50.828Z+ b=crutest
1112024-02-27T21:43:50.828Z+ gunzip
1122024-02-27T21:43:53.858Z+ chmod +x /var/tmp/bins/crutest
1132024-02-27T21:43:53.859Z+ for t in "$input/bins/"*.gz
1142024-02-27T21:43:53.859Z++ basename /input/build/work/bins/dsc.gz
1152024-02-27T21:43:53.859Z+ b=dsc.gz
1162024-02-27T21:43:53.859Z+ b=dsc
1172024-02-27T21:43:53.859Z+ gunzip
1182024-02-27T21:43:54.927Z+ chmod +x /var/tmp/bins/dsc
1192024-02-27T21:43:54.949Z+ export BINDIR=/var/tmp/bins
1202024-02-27T21:43:54.949Z+ BINDIR=/var/tmp/bins
1212024-02-27T21:43:54.949Z+ banner repair
1222024-02-27T21:43:54.949Z
1232024-02-27T21:43:54.949Z ##### ###### ##### ## # #####
1242024-02-27T21:43:54.950Z # # # # # # # # # #
1252024-02-27T21:43:54.950Z # # ##### # # # # # # #
1262024-02-27T21:43:54.950Z ##### # ##### ###### # #####
1272024-02-27T21:43:54.951Z # # # # # # # # #
1282024-02-27T21:43:54.951Z # # ###### # # # # # #
1292024-02-27T21:43:54.951Z
1302024-02-27T21:43:54.951Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-27T21:43:55.060ZTurn off color for downstairs dump
1322024-02-27T21:43:55.075ZCreating region directory at: /var/tmp/test_repair
1332024-02-27T21:43:55.076ZCreating dsc directory at: /tmp/dsc
1342024-02-27T21:43:55.168ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.09443873
1352024-02-27T21:43:55.618ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.042438913
1362024-02-27T21:43:55.651ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.037430312
1372024-02-27T21:43:55.651ZRegion set with 3 regions was created
1382024-02-27T21:43:55.651Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1140 1141 1142
1392024-02-27T21:43:55.652ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-27T21:43:55.652Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.697112195Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144}
1412024-02-27T21:43:55.652Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.697942517Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144}
1422024-02-27T21:43:55.652Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.697972383Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144}
1432024-02-27T21:43:55.652Z{"msg":"Crucible 5bf77e4c-1d82-40a6-9ede-61b2c5788acb has session id: fd21f0cb-446b-4d89-8ffc-d269803eda6f","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.69799518Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1442024-02-27T21:43:55.652Z{"msg":"Upstairs opts: Upstairs UUID: 5bf77e4c-1d82-40a6-9ede-61b2c5788acb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.698043835Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1452024-02-27T21:43:55.652Z{"msg":"Crucible stats registered with UUID: 5bf77e4c-1d82-40a6-9ede-61b2c5788acb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.698061373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1462024-02-27T21:43:55.653ZCrucible runtime is spawned
1472024-02-27T21:43:55.653Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.698996113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144}
1482024-02-27T21:43:55.653Z{"msg":"5bf77e4c-1d82-40a6-9ede-61b2c5788acb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699062125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1492024-02-27T21:43:55.653Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699088562Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1502024-02-27T21:43:55.653Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699125467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1512024-02-27T21:43:55.653Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699141875Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1522024-02-27T21:43:55.653Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699156134Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1532024-02-27T21:43:55.653Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699183651Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1542024-02-27T21:43:55.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699204508Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1552024-02-27T21:43:55.654Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699228245Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1562024-02-27T21:43:55.654Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699262751Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1572024-02-27T21:43:55.654Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699363289Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1582024-02-27T21:43:55.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699492764Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1592024-02-27T21:43:55.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699535579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1602024-02-27T21:43:55.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.699558396Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1612024-02-27T21:43:55.654Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.701361034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1622024-02-27T21:43:55.654Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.701431656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1632024-02-27T21:43:55.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.70147729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1642024-02-27T21:43:55.655Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.701533404Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1652024-02-27T21:43:55.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.701556391Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1662024-02-27T21:43:55.655Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.701583948Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1672024-02-27T21:43:55.655Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.703196748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1682024-02-27T21:43:55.655Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.703235273Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1692024-02-27T21:43:55.655Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.703257231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1702024-02-27T21:43:55.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704045758Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1712024-02-27T21:43:55.655Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704125958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1722024-02-27T21:43:55.655Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.70419328Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1732024-02-27T21:43:55.656Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704229736Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1742024-02-27T21:43:55.656Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704246974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1752024-02-27T21:43:55.656Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704269221Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1762024-02-27T21:43:55.656Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.70428231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1772024-02-27T21:43:55.656Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704295898Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1782024-02-27T21:43:55.656Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704316946Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1792024-02-27T21:43:55.656Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704331454Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1802024-02-27T21:43:55.656Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704438332Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1812024-02-27T21:43:55.656Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704465488Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1822024-02-27T21:43:55.656Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704480007Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1832024-02-27T21:43:55.656Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704500904Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1842024-02-27T21:43:55.656Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704515472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1852024-02-27T21:43:55.656Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.70453568Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1862024-02-27T21:43:55.657Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704550308Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1872024-02-27T21:43:55.657Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704570306Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1882024-02-27T21:43:55.657Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704584224Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1892024-02-27T21:43:55.657Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704604262Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1902024-02-27T21:43:55.657Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704679143Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"downstairs","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1912024-02-27T21:43:55.657Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.70470493Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1922024-02-27T21:43:55.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704718008Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1932024-02-27T21:43:55.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704731677Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"1","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1942024-02-27T21:43:55.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
1952024-02-27T21:43:55.658Z2024-02-27T21:43:53.704752484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"client":"2","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1962024-02-27T21:43:55.658Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704769122Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1972024-02-27T21:43:55.658Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.70478813Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1982024-02-27T21:43:55.658Z{"msg":"5bf77e4c-1d82-40a6-9ede-61b2c5788acb is now active with session: fd21f0cb-446b-4d89-8ffc-d269803eda6f","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704801919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
1992024-02-27T21:43:55.658Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704822116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144}
2002024-02-27T21:43:55.658Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:53.704833475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
2012024-02-27T21:43:55.658ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-27T21:43:55.659ZFill test
2032024-02-27T21:43:55.659ZRead and Verify all blocks (0..600 range:false)
2042024-02-27T21:43:55.666ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-27T21:43:55.666ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-27T21:43:55.679Z----------------------------------------------------------------
2072024-02-27T21:43:55.679Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-27T21:43:55.679ZStates: Active Active Active
2092024-02-27T21:43:55.679ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-27T21:43:55.679Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-27T21:43:55.679Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-27T21:43:55.679Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-27T21:43:55.680Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-27T21:43:55.680Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-27T21:43:55.680Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-27T21:43:55.680Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-27T21:43:55.680Z New 0 0 0 0
2182024-02-27T21:43:55.680Z Sent 0 0 0 0
2192024-02-27T21:43:55.681Z Done 6 6 6 18
2202024-02-27T21:43:55.681Z Skipped 0 0 0 0
2212024-02-27T21:43:55.681Z Error 0 0 0 0
2222024-02-27T21:43:55.681ZLast Flush: 1006 1006 1006
2232024-02-27T21:43:55.681ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-27T21:43:55.681ZUpstairs last five completed: 13 12 11 10 9
2252024-02-27T21:43:55.682ZCLIENT: Up:0 ds:6 act:3
2262024-02-27T21:43:59.679Z----------------------------------------------------------------
2272024-02-27T21:43:59.679Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-27T21:43:59.679ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-27T21:43:59.679ZUpstairs last five completed: 14 13 12 11 10
2302024-02-27T21:43:59.680ZCLIENT: Up:0 ds:0 act:3
2312024-02-27T21:43:59.680ZCLIENT: All crucible jobs finished, exiting program
2322024-02-27T21:44:00.245Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:43:58.084996636Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144}
2332024-02-27T21:44:00.245Z{"msg":"client_stop_rx closed unexpectedly: RecvError(())","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:43:58.085359114Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
2342024-02-27T21:44:00.245Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:43:58.085412757Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
2352024-02-27T21:44:00.245Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:43:58.085448643Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
2362024-02-27T21:44:00.245Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.08547655Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1144,"":"io task","client":"0","":"downstairs","session_id":"fd21f0cb-446b-4d89-8ffc-d269803eda6f"}
2372024-02-27T21:44:00.245Z
2382024-02-27T21:44:00.246ZBegin loop 1
2392024-02-27T21:44:00.246ZDownstairs to restart: 1
2402024-02-27T21:44:00.246Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108075428Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146}
2412024-02-27T21:44:00.246Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108343726Z","hostname":"Crucible runtime is spawned
2422024-02-27T21:44:00.246Zip-10-150-1-196.us-west-2.compute.internal","pid":1146}
2432024-02-27T21:44:00.246Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108410388Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146}
2442024-02-27T21:44:00.246Z{"msg":"Crucible 69348cfc-f78d-4713-8352-6852777edd8c has session id: 5f9c0206-5863-44fd-adcf-de76a0c36ceb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108453523Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2452024-02-27T21:44:00.246Z{"msg":"Upstairs opts: Upstairs UUID: 69348cfc-f78d-4713-8352-6852777edd8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108493149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2462024-02-27T21:44:00.247Z{"msg":"Crucible stats registered with UUID: 69348cfc-f78d-4713-8352-6852777edd8c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108527035Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2472024-02-27T21:44:00.247Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108549942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146}
2482024-02-27T21:44:00.247Z{"msg":"69348cfc-f78d-4713-8352-6852777edd8c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108572009Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2492024-02-27T21:44:00.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108593777Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2502024-02-27T21:44:00.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108619554Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2512024-02-27T21:44:00.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.1086528Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2522024-02-27T21:44:00.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108676017Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2532024-02-27T21:44:00.248Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108698254Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2542024-02-27T21:44:00.248Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108723042Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2552024-02-27T21:44:00.248Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108746689Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"io task","client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2562024-02-27T21:44:00.248Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108789164Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"io task","client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2572024-02-27T21:44:00.248Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108828749Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"io task","client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2582024-02-27T21:44:00.248Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.108866355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"io task","client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2592024-02-27T21:44:00.248Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.109022746Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"io task","client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2602024-02-27T21:44:00.249Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.109070991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"io task","client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2612024-02-27T21:44:00.249Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.109537556Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2622024-02-27T21:44:00.249Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.109598368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2632024-02-27T21:44:00.249Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.109675919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2642024-02-27T21:44:00.249Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.109738582Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2652024-02-27T21:44:00.249Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.109776787Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2662024-02-27T21:44:00.249Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.109799725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2672024-02-27T21:44:00.249Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.110224365Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2682024-02-27T21:44:00.249Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.11026111Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2692024-02-27T21:44:00.249Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.11034948Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2702024-02-27T21:44:00.262Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.110765291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2712024-02-27T21:44:00.262Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.110823374Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2722024-02-27T21:44:00.262Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.110850341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2732024-02-27T21:44:00.262Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.110949589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2742024-02-27T21:44:00.262Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.110983225Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2752024-02-27T21:44:00.262Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111011342Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2762024-02-27T21:44:00.262Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.11103295Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2772024-02-27T21:44:00.263Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111047918Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2782024-02-27T21:44:00.263Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111062726Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2792024-02-27T21:44:00.263Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111083924Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2802024-02-27T21:44:00.263Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111098702Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2812024-02-27T21:44:00.263Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111119959Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2822024-02-27T21:44:00.263Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111134018Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2832024-02-27T21:44:00.263Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111154845Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2842024-02-27T21:44:00.263Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111169203Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2852024-02-27T21:44:00.263Z{"msg":"Wait for a query_work_queue command to finish before sending IO
2862024-02-27T21:44:00.263Z[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111192221Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2872024-02-27T21:44:00.263Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111206579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2882024-02-27T21:44:00.264Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111226537Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2892024-02-27T21:44:00.264Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111240175Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2902024-02-27T21:44:00.264Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111259443Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2912024-02-27T21:44:00.264Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111272831Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"":"downstairs","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2922024-02-27T21:44:00.264Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111292669Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2932024-02-27T21:44:00.264Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111305098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"0","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2942024-02-27T21:44:00.264Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111318256Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"1","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2952024-02-27T21:44:00.264Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111338194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"client":"2","":"downstairs","session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2962024-02-27T21:44:00.264Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111351932Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2972024-02-27T21:44:00.264Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.11137104Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2982024-02-27T21:44:00.264Z{"msg":"69348cfc-f78d-4713-8352-6852777edd8c is now active with session: 5f9c0206-5863-44fd-adcf-de76a0c36ceb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111384898Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
2992024-02-27T21:44:00.264Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111397647Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146,"session_id":"5f9c0206-5863-44fd-adcf-de76a0c36ceb"}
3002024-02-27T21:44:00.264Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:43:58.111417414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146}
3012024-02-27T21:44:00.264ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
3022024-02-27T21:44:00.264ZLoading write count information from file "/tmp/test_repair_verify.data"
3032024-02-27T21:44:00.265ZRun Repair workload
3042024-02-27T21:44:00.265Z01/30 Write block 450 len 16384 data: 2 2 2 2
3052024-02-27T21:44:00.265Z02/30 Read block 535 len 8192
3062024-02-27T21:44:00.265Z03/30 Read block 482 len 28672
3072024-02-27T21:44:00.265Z04/30 Read block 506 len 32768
3082024-02-27T21:44:00.265Z05/30 Flush
3092024-02-27T21:44:00.265Z06/30 Write block 4 len 32768 data: 2 2 2 2 2 2 2 2
3102024-02-27T21:44:00.265Z07/30 Read block 446 len 8192
3112024-02-27T21:44:00.265Z08/30 Write block 358 len 12288 data: 2 2 2
3122024-02-27T21:44:00.265Z09/30 Write block 189 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3132024-02-27T21:44:00.265Z10/30 Write block 38 len 32768 data: 2 2 2 2 2 2 2 2
3142024-02-27T21:44:00.265Z11/30 Write block 146 len 32768 data: 2 2 2 2 2 2 2 2
3152024-02-27T21:44:00.265Z12/30 Read block 405 len 16384
3162024-02-27T21:44:00.266Z13/30 Write block 443 len 24576 data: 2 2 2 2 2 2
3172024-02-27T21:44:00.266Z14/30 Flush
3182024-02-27T21:44:00.266Z15/30 Write block 387 len 28672 data: 2 2 2 2 2 2 2
3192024-02-27T21:44:00.266Z16/30 Read block 210 len 32768
3202024-02-27T21:44:00.266Z17/30 Write block 489 len 24576 data: 2 2 2 2 2 2
3212024-02-27T21:44:00.266Z18/30 Write block 100 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3222024-02-27T21:44:00.266Z19/30 Write block 226 len 8192 data: 2 2
3232024-02-27T21:44:00.266Z20/30 Write block 492 len 36864 data: 3 3 3 2 2 2 2 2 2
3242024-02-27T21:44:00.266Z21/30 Read block 99 len 32768
3252024-02-27T21:44:00.266Z22/30 Write block 387 len 12288 data: 3 3 3
3262024-02-27T21:44:00.266Z23/30 Flush
3272024-02-27T21:44:00.266Z24/30 Write block 406 len 36864 data: 2 2 2 2 2 2 2 2 2
3282024-02-27T21:44:00.266Z25/30 Write block 596 len 12288 data: 2 2 2
3292024-02-27T21:44:00.266Z26/30 Read block 157 len 4096
3302024-02-27T21:44:00.267Z27/30 Write block 263 len 8192 data: 2 2
3312024-02-27T21:44:00.267Z28/30 Read block 84 len 36864
3322024-02-27T21:44:00.267Z29/30 Read block 382 len 16384
3332024-02-27T21:44:00.267Z30/30 Write block 339 len 4096 data: 2
3342024-02-27T21:44:00.267Z----------------------------------------------------------------
3352024-02-27T21:44:00.267Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3362024-02-27T21:44:00.267ZStates: Active Active Active
3372024-02-27T21:44:00.267ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3382024-02-27T21:44:00.267Z 1 Acked 1000 Write 4 Done Done Done false
3392024-02-27T21:44:00.267Z 2 Acked 1001 Read 2 Done Done Done false
3402024-02-27T21:44:00.267Z 3 Acked 1002 Read 7 Done Done Done false
3412024-02-27T21:44:00.267Z 4 Acked 1003 Read 8 Done Done Done false
3422024-02-27T21:44:00.267Z 5 Acked 1004 Flush 0 Done Sent Done false
3432024-02-27T21:44:00.268Z 6 Acked 1005 Write 8 Done Sent Done false
3442024-02-27T21:44:00.268Z 7 Acked 1006 Read 2 Done Sent Done false
3452024-02-27T21:44:00.268Z 8 Acked 1007 Write 3 Done Sent Done false
3462024-02-27T21:44:00.268Z 9 Acked 1008 Write 10 Done Sent Done false
3472024-02-27T21:44:00.268Z 10 Acked 1009 Write 8 Done Sent Done false
3482024-02-27T21:44:00.268Z 11 Acked 1010 Write 8 Done Sent Done false
3492024-02-27T21:44:00.268Z 12 Acked 1011 Read 4 Done Sent Done false
3502024-02-27T21:44:00.268Z 13 Acked 1012 Write 6 Done Sent Done false
3512024-02-27T21:44:00.268Z 14 Acked 1013 Flush 0 Done Sent Done false
3522024-02-27T21:44:00.268Z 15 Acked 1014 Write 7 Done Sent Done false
3532024-02-27T21:44:00.268Z 16 Acked 1015 Read 8 Done Sent Done false
3542024-02-27T21:44:00.268Z 17 Acked 1016 Write 6 Done Sent Done false
3552024-02-27T21:44:00.268Z 18 Acked 1017 Write 10 Done Sent Done false
3562024-02-27T21:44:00.268Z 19 Acked 1018 Write 2 Done Sent Done false
3572024-02-27T21:44:00.268Z 20 Acked 1019 Write 9 Done Sent Done false
3582024-02-27T21:44:00.268Z 21 Acked 1020 Read 8 Done Sent Done false
3592024-02-27T21:44:00.268Z 22 Acked 1021 Write 3 Done Sent Done false
3602024-02-27T21:44:00.268Z 23 Acked 1022 Flush 0 Done Sent Done false
3612024-02-27T21:44:00.269Z 24 Acked 1023 Write 9 Done Sent Done false
3622024-02-27T21:44:00.269Z 25 Acked 1024 Write 3 Done Sent Done false
3632024-02-27T21:44:00.269Z 26 Acked 1025 Read 1 Done Sent Done false
3642024-02-27T21:44:00.269Z 27 Acked 1026 Write 2 Done Sent Done false
3652024-02-27T21:44:00.269Z 28 Acked 1027 Read 9 Done Sent Done false
3662024-02-27T21:44:00.269Z 29 Acked 1028 Read 4 Done Sent Done false
3672024-02-27T21:44:00.269Z 30 Acked 1029 Write 1 Sent Sent Sent false
3682024-02-27T21:44:00.269Z STATES DS:0 DS:1 DS:2 TOTAL
3692024-02-27T21:44:00.269Z New 0 0 0 0
3702024-02-27T21:44:00.282Z Sent 1 26 1 28
3712024-02-27T21:44:00.282Z Done 29 4 29 62
3722024-02-27T21:44:00.282Z Skipped 0 0 0 0
3732024-02-27T21:44:00.282Z Error 0 0 0 0
3742024-02-27T21:44:00.282ZLast Flush: 1022 0 1022
3752024-02-27T21:44:00.282ZDownstairs last five completed:
3762024-02-27T21:44:00.282ZUpstairs last five completed: 30 29 28 27 26
3772024-02-27T21:44:00.283Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:43:58.324019502Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1146}
3782024-02-27T21:44:00.283ZWrote out file "/tmp/test_repair_verify.data"
3792024-02-27T21:44:00.283Z
3802024-02-27T21:44:00.283ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3812024-02-27T21:44:00.283ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3822024-02-27T21:44:00.283Z 0 000-019 2 1 2 3 1 3 F F F <---
3832024-02-27T21:44:00.283Z 1 020-039 2 1 2 3 1 3 F F F <---
3842024-02-27T21:44:00.283Z 2 040-059 2 1 2 3 1 3 F F F <---
3852024-02-27T21:44:00.283Z 3 060-079 1 1 1 1 1 1 F F F
3862024-02-27T21:44:00.283Z 4 080-099 1 1 1 1 1 1 F F F
3872024-02-27T21:44:00.283Z 5 100-119 2 1 2 4 1 4 F F F <---
3882024-02-27T21:44:00.283Z 6 120-139 1 1 1 1 1 1 F F F
3892024-02-27T21:44:00.283Z 7 140-159 2 1 2 3 1 3 F F F <---
3902024-02-27T21:44:00.283Z 8 160-179 1 1 1 1 1 1 F F F
3912024-02-27T21:44:00.283Z 9 180-199 2 1 2 3 1 3 F F F <---
3922024-02-27T21:44:00.283Z 10 200-219 1 1 1 1 1 1 F F F
3932024-02-27T21:44:00.283Z 11 220-239 2 1 2 4 1 4 F F F <---
3942024-02-27T21:44:00.283Z 12 240-259 1 1 1 1 1 1 F F F
3952024-02-27T21:44:00.284Z 13 260-279 1 1 1 1 1 1 T F T <---
3962024-02-27T21:44:00.284Z 14 280-299 1 1 1 1 1 1 F F F
3972024-02-27T21:44:00.284Z 15 300-319 1 1 1 1 1 1 F F F
3982024-02-27T21:44:00.284Z 16 320-339 1 1 1 1 1 1 T F T <---
3992024-02-27T21:44:00.284Z 17 340-359 2 1 2 3 1 3 F F F <---
4002024-02-27T21:44:00.284Z 18 360-379 2 1 2 3 1 3 F F F <---
4012024-02-27T21:44:00.284Z 19 380-399 2 1 2 4 1 4 F F F <---
4022024-02-27T21:44:00.284Z 20 400-419 1 1 1 1 1 1 T F T <---
4032024-02-27T21:44:00.284Z 21 420-439 1 1 1 1 1 1 F F F
4042024-02-27T21:44:00.284Z 22 440-459 2 1 2 3 1 3 F T F <---
4052024-02-27T21:44:00.284Z 23 460-479 1 1 1 1 1 1 F F F
4062024-02-27T21:44:00.284Z 24 480-499 2 1 2 4 1 4 F F F <---
4072024-02-27T21:44:00.284Z 25 500-519 2 1 2 4 1 4 F F F <---
4082024-02-27T21:44:00.284Z 26 520-539 1 1 1 1 1 1 F F F
4092024-02-27T21:44:00.284Z 27 540-559 1 1 1 1 1 1 F F F
4102024-02-27T21:44:00.284Z 28 560-579 1 1 1 1 1 1 F F F
4112024-02-27T21:44:00.284Z 29 580-599 1 1 1 1 1 1 T F T <---
4122024-02-27T21:44:00.284ZMax gen: 2, Max flush: 4
4132024-02-27T21:44:00.284ZError: Difference in extent metadata found!
4142024-02-27T21:44:00.284ZA Difference in extent metadata is expected here
4152024-02-27T21:44:00.284ZOn loop 1
4162024-02-27T21:44:00.285Z
4172024-02-27T21:44:00.285ZVerifying data now
4182024-02-27T21:44:11.423ZNo change after verify
4192024-02-27T21:44:11.423ZLoop: 1 Downstairs dump after verify (and repair):
4202024-02-27T21:44:11.445ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4212024-02-27T21:44:11.445Z 0 000-019 2 2 2 3 3 3 F F F
4222024-02-27T21:44:11.445Z 1 020-039 2 2 2 3 3 3 F F F
4232024-02-27T21:44:11.446Z 2 040-059 2 2 2 3 3 3 F F F
4242024-02-27T21:44:11.446Z 3 060-079 1 1 1 1 1 1 F F F
4252024-02-27T21:44:11.446Z 4 080-099 1 1 1 1 1 1 F F F
4262024-02-27T21:44:11.446Z 5 100-119 2 2 2 4 4 4 F F F
4272024-02-27T21:44:11.446Z 6 120-139 1 1 1 1 1 1 F F F
4282024-02-27T21:44:11.446Z 7 140-159 2 2 2 3 3 3 F F F
4292024-02-27T21:44:11.446Z 8 160-179 1 1 1 1 1 1 F F F
4302024-02-27T21:44:11.446Z 9 180-199 2 2 2 3 3 3 F F F
4312024-02-27T21:44:11.487Z 10 200-219 1 1 1 1 1 1 F F F
4322024-02-27T21:44:11.487Z 11 220-239 2 2 2 4 4 4 F F F
4332024-02-27T21:44:11.487Z 12 240-259 1 1 1 1 1 1 F F F
4342024-02-27T21:44:11.487Z 13 260-279 3 3 3 5 5 5 F F F
4352024-02-27T21:44:11.487Z 14 280-299 1 1 1 1 1 1 F F F
4362024-02-27T21:44:11.487Z 15 300-319 1 1 1 1 1 1 F F F
4372024-02-27T21:44:11.487Z 16 320-339 3 3 3 5 5 5 F F F
4382024-02-27T21:44:11.487Z 17 340-359 2 2 2 3 3 3 F F F
4392024-02-27T21:44:11.488Z 18 360-379 2 2 2 3 3 3 F F F
4402024-02-27T21:44:11.488Z 19 380-399 2 2 2 4 4 4 F F F
4412024-02-27T21:44:11.488Z 20 400-419 3 3 3 5 5 5 F F F
4422024-02-27T21:44:11.488Z 21 420-439 1 1 1 1 1 1 F F F
4432024-02-27T21:44:11.488Z 22 440-459 2 2 2 3 3 3 F F F
4442024-02-27T21:44:11.488Z 23 460-479 1 1 1 1 1 1 F F F
4452024-02-27T21:44:11.488Z 24 480-499 2 2 2 4 4 4 F F F
4462024-02-27T21:44:11.488Z 25 500-519 2 2 2 4 4 4 F F F
4472024-02-27T21:44:11.488Z 26 520-539 1 1 1 1 1 1 F F F
4482024-02-27T21:44:11.489Z 27 540-559 1 1 1 1 1 1 F F F
4492024-02-27T21:44:11.489Z 28 560-579 1 1 1 1 1 1 F F F
4502024-02-27T21:44:11.489Z 29 580-599 3 3 3 5 5 5 F F F
4512024-02-27T21:44:11.489ZMax gen: 3, Max flush: 5
4522024-02-27T21:44:11.489Z
4532024-02-27T21:44:11.489ZBegin loop 2
4542024-02-27T21:44:11.490ZDownstairs to restart: 2
4552024-02-27T21:44:11.510Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873194997Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154}
4562024-02-27T21:44:11.510ZCrucible runtime is spawned
4572024-02-27T21:44:11.511Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873518619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154}
4582024-02-27T21:44:11.511Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873554885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154}
4592024-02-27T21:44:11.511Z{"msg":"Crucible e3073e17-f0e5-4ec6-b206-8f29f493ded0 has session id: a65cdc78-d8ca-48e8-80d3-05a959ea3f30","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.87359703Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4602024-02-27T21:44:11.512Z{"msg":"Upstairs opts: Upstairs UUID: e3073e17-f0e5-4ec6-b206-8f29f493ded0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873634755Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4612024-02-27T21:44:11.512Z{"msg":"Crucible stats registered with UUID: e3073e17-f0e5-4ec6-b206-8f29f493ded0","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.87367815Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4622024-02-27T21:44:11.512Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873704707Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154}
4632024-02-27T21:44:11.512Z{"msg":"e3073e17-f0e5-4ec6-b206-8f29f493ded0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873724005Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4642024-02-27T21:44:11.512Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873753631Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4652024-02-27T21:44:11.513Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873782518Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4662024-02-27T21:44:11.513Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873821083Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4672024-02-27T21:44:11.513Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.87384772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4682024-02-27T21:44:11.513Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873873287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4692024-02-27T21:44:11.513Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873904543Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4702024-02-27T21:44:11.513Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873927631Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"io task","client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4712024-02-27T21:44:11.514Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.873968186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"io task","client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4722024-02-27T21:44:11.514Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.874051616Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"io task","client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4732024-02-27T21:44:11.514Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.874096631Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"io task","client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4742024-02-27T21:44:11.514Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.87435685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"io task","client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4752024-02-27T21:44:11.514Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.874398825Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"io task","client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4762024-02-27T21:44:11.514Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.874802888Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4772024-02-27T21:44:11.515Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.874843103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4782024-02-27T21:44:11.515Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.874933882Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4792024-02-27T21:44:11.515Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.874967278Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4802024-02-27T21:44:11.515Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.874992326Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4812024-02-27T21:44:11.515Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.87503674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4822024-02-27T21:44:11.516Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.875283011Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4832024-02-27T21:44:11.516Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.875432064Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4842024-02-27T21:44:11.516Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.875664026Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4852024-02-27T21:44:11.516Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.875854144Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4862024-02-27T21:44:11.516Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.875904478Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4872024-02-27T21:44:11.516Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.875954542Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4882024-02-27T21:44:11.517Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.875978199Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4892024-02-27T21:44:11.517Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876026744Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4902024-02-27T21:44:11.517Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876064369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4912024-02-27T21:44:11.517Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876134851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4922024-02-27T21:44:11.517Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876179176Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4932024-02-27T21:44:11.517Z{"msg":"[0]R flush_numbers[0..12]: [3, 3, 3, 1, 1, 4, 1, 3, 1, 3, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876202043Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4942024-02-27T21:44:11.518Z{"msg":"[0]R generation[0..12]: [2, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876234029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4952024-02-27T21:44:11.518Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876251747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4962024-02-27T21:44:11.518Z{"msg":"[1]R flush_numbers[0..12]: [3, 3, 3, 1, 1, 4, 1, 3, 1, 3, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876283273Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4972024-02-27T21:44:11.518Z{"msg":"[1]R generation[0..12]: [2, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876320389Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
4982024-02-27T21:44:11.518Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","Wait for a query_work_queue command to finish before sending IO
4992024-02-27T21:44:11.519Zv":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876350785Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5002024-02-27T21:44:11.519Z{"msg":"[2]R flush_numbers[0..12]: [3, 3, 3, 1, 1, 4, 1, 3, 1, 3, 1, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876384731Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5012024-02-27T21:44:11.519Z{"msg":"[2]R generation[0..12]: [2, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876409959Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5022024-02-27T21:44:11.519Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876438955Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5032024-02-27T21:44:11.519Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876470641Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5042024-02-27T21:44:11.519Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876507497Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5052024-02-27T21:44:11.519Z{"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876531194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5062024-02-27T21:44:11.519Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876556961Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"":"downstairs","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5072024-02-27T21:44:11.520Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876584988Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5082024-02-27T21:44:11.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876616314Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"0","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5092024-02-27T21:44:11.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876635562Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"1","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5102024-02-27T21:44:11.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5112024-02-27T21:44:11.520Z,"time":"2024-02-27T21:44:09.876667488Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"client":"2","":"downstairs","session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5122024-02-27T21:44:11.521Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876695855Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5132024-02-27T21:44:11.521Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876725971Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5142024-02-27T21:44:11.521Z{"msg":"e3073e17-f0e5-4ec6-b206-8f29f493ded0 is now active with session: a65cdc78-d8ca-48e8-80d3-05a959ea3f30","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.87674092Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5152024-02-27T21:44:11.521Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876760727Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154,"session_id":"a65cdc78-d8ca-48e8-80d3-05a959ea3f30"}
5162024-02-27T21:44:11.521Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:09.876773366Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154}
5172024-02-27T21:44:11.521ZLoading write count information from file "/tmp/test_repair_verify.data"
5182024-02-27T21:44:11.522ZRun Repair workload
5192024-02-27T21:44:11.522Z01/30 Write block 106 len 28672 data: 3 3 3 3 2 2 2
5202024-02-27T21:44:11.522Z02/30 Write block 116 len 32768 data: 2 2 2 2 2 2 2 2
5212024-02-27T21:44:11.522Z03/30 Read block 283 len 20480
5222024-02-27T21:44:11.522Z04/30 Write block 462 len 4096 data: 2
5232024-02-27T21:44:11.522Z05/30 Flush
5242024-02-27T21:44:11.522Z06/30 Write block 337 len 32768 data: 2 2 3 2 2 2 2 2
5252024-02-27T21:44:11.522Z07/30 Write block 491 len 8192 data: 3 4
5262024-02-27T21:44:11.522Z08/30 Flush
5272024-02-27T21:44:11.522Z09/30 Write block 27 len 20480 data: 2 2 2 2 2
5282024-02-27T21:44:11.523Z10/30 Write block 569 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5292024-02-27T21:44:11.523Z11/30 Write block 489 len 24576 data: 3 3 4 5 4 4
5302024-02-27T21:44:11.523Z12/30 Read block 132 len 4096
5312024-02-27T21:44:11.523Z13/30 Read block 591 len 36864
5322024-02-27T21:44:11.523Z14/30 Read block 433 len 12288
5332024-02-27T21:44:11.523Z15/30 Read block 499 len 40960
5342024-02-27T21:44:11.523Z16/30 Write block 135 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5352024-02-27T21:44:11.524Z17/30 Write block 267 len 20480 data: 2 2 2 2 2
5362024-02-27T21:44:11.524Z18/30 Read block 436 len 20480
5372024-02-27T21:44:11.524Z19/30 Read block 435 len 40960
5382024-02-27T21:44:11.524Z20/30 Write block 309 len 32768 data: 2 2 2 2 2 2 2 2
5392024-02-27T21:44:11.524Z21/30 Read block 334 len 12288
5402024-02-27T21:44:11.524Z22/30 Read block 149 len 24576
5412024-02-27T21:44:11.524Z23/30 Write block 283 len 4096 data: 2
5422024-02-27T21:44:11.524Z24/30 Read block 556 len 16384
5432024-02-27T21:44:11.524Z25/30 Flush
5442024-02-27T21:44:11.537Z26/30 Write block 43 len 20480 data: 3 3 3 2 2
5452024-02-27T21:44:11.537Z27/30 Write block 289 len 8192 data: 2 2
5462024-02-27T21:44:11.537Z28/30 Write block 200 len 32768 data: 2 2 2 2 2 2 2 2
5472024-02-27T21:44:11.537Z29/30 Read block 561 len 24576
5482024-02-27T21:44:11.537Z30/30 Read block 81 len 4096
5492024-02-27T21:44:11.538Z----------------------------------------------------------------
5502024-02-27T21:44:11.538Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:22
5512024-02-27T21:44:11.538ZStates: Active Active Active
5522024-02-27T21:44:11.538ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5532024-02-27T21:44:11.538Z 9 Acked 1008 Write 5 Done Done Done false
5542024-02-27T21:44:11.538Z 10 Acked 1009 Write 10 Done Done Done false
5552024-02-27T21:44:11.538Z 11 Acked 1010 Write 6 Done Done Done false
5562024-02-27T21:44:11.538Z 12 Acked 1011 Read 1 Done Done Done false
5572024-02-27T21:44:11.538Z 13 Acked 1012 Read 9 Done Done Sent false
5582024-02-27T21:44:11.538Z 14 Acked 1013 Read 3 Done Done Sent false
5592024-02-27T21:44:11.539Z 15 Acked 1014 Read 10 Done Done Sent false
5602024-02-27T21:44:11.539Z 16 Acked 1015 Write 10 Done Done Sent false
5612024-02-27T21:44:11.539Z 17 Acked 1016 Write 5 Done Done Sent false
5622024-02-27T21:44:11.539Z 18 Acked 1017 Read 5 Done Done Sent false
5632024-02-27T21:44:11.539Z 19 Acked 1018 Read 10 Done Done Sent false
5642024-02-27T21:44:11.539Z 20 Acked 1019 Write 8 Done Done Sent false
5652024-02-27T21:44:11.539Z 21 Acked 1020 Read 3 Done Done Sent false
5662024-02-27T21:44:11.539Z 22 Acked 1021 Read 6 Done Done Sent false
5672024-02-27T21:44:11.539Z 23 Acked 1022 Write 1 Done Done Sent false
5682024-02-27T21:44:11.539Z 24 Acked 1023 Read 4 Done Done Sent false
5692024-02-27T21:44:11.540Z 25 Acked 1024 Flush 0 Done Done Sent false
5702024-02-27T21:44:11.540Z 26 Acked 1025 Write 5 Done Done Sent false
5712024-02-27T21:44:11.540Z 27 Acked 1026 Write 2 Done Done Sent false
5722024-02-27T21:44:11.540Z 28 Acked 1027 Write 8 Done Done Sent false
5732024-02-27T21:44:11.540Z 29 Acked 1028 Read 6 Done Done Sent false
5742024-02-27T21:44:11.540Z 30 Acked 1029 Read 1 Done Done Sent false
5752024-02-27T21:44:11.540Z STATES DS:0 DS:1 DS:2 TOTAL
5762024-02-27T21:44:11.540Z New 0 0 0 0
5772024-02-27T21:44:11.540Z Sent 0 0 18 18
5782024-02-27T21:44:11.540Z Done 22 22 4 48
5792024-02-27T21:44:11.540Z Skipped 0 0 0 0
5802024-02-27T21:44:11.541Z Error 0 0 0 0
5812024-02-27T21:44:11.541ZLast Flush: 1024 1024 1007
5822024-02-27T21:44:11.541ZDownstairs last five completed: 1007 1006 1005 1004 1003
5832024-02-27T21:44:11.541ZUpstairs last five completed: 30 29 28 27 26
5842024-02-27T21:44:11.541Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:09.920951292Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1154}
5852024-02-27T21:44:11.541ZWrote out file "/tmp/test_repair_verify.data"
5862024-02-27T21:44:11.541Z
5872024-02-27T21:44:11.541ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5882024-02-27T21:44:11.541ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5892024-02-27T21:44:11.541Z 0 000-019 2 2 2 3 3 3 F F F
5902024-02-27T21:44:11.541Z 1 020-039 4 4 2 8 8 3 F F T <---
5912024-02-27T21:44:11.541Z 2 040-059 2 2 2 3 3 3 T T F <---
5922024-02-27T21:44:11.542Z 3 060-079 1 1 1 1 1 1 F F F
5932024-02-27T21:44:11.542Z 4 080-099 1 1 1 1 1 1 F F F
5942024-02-27T21:44:11.542Z 5 100-119 4 4 4 6 6 6 F F F
5952024-02-27T21:44:11.542Z 6 120-139 4 4 4 8 8 6 F F F <---
5962024-02-27T21:44:11.542Z 7 140-159 4 4 2 8 8 3 F F F <---
5972024-02-27T21:44:11.542Z 8 160-179 1 1 1 1 1 1 F F F
5982024-02-27T21:44:11.542Z 9 180-199 2 2 2 3 3 3 F F F
5992024-02-27T21:44:11.542Z 10 200-219 1 1 1 1 1 1 T T F <---
6002024-02-27T21:44:11.543Z 11 220-239 2 2 2 4 4 4 F F F
6012024-02-27T21:44:11.543Z 12 240-259 1 1 1 1 1 1 F F F
6022024-02-27T21:44:11.543Z 13 260-279 4 4 3 8 8 5 F F F <---
6032024-02-27T21:44:11.543Z 14 280-299 4 4 1 8 8 1 T T F <---
6042024-02-27T21:44:11.543Z 15 300-319 4 4 1 8 8 1 F F F <---
6052024-02-27T21:44:11.543Z 16 320-339 4 4 4 7 7 7 F F F
6062024-02-27T21:44:11.543Z 17 340-359 4 4 4 7 7 7 F F F
6072024-02-27T21:44:11.543Z 18 360-379 2 2 2 3 3 3 F F F
6082024-02-27T21:44:11.543Z 19 380-399 2 2 2 4 4 4 F F F
6092024-02-27T21:44:11.543Z 20 400-419 3 3 3 5 5 5 F F F
6102024-02-27T21:44:11.543Z 21 420-439 1 1 1 1 1 1 F F F
6112024-02-27T21:44:11.544Z 22 440-459 2 2 2 3 3 3 F F F
6122024-02-27T21:44:11.544Z 23 460-479 4 4 4 6 6 6 F F F
6132024-02-27T21:44:11.544Z 24 480-499 4 4 4 8 8 7 F F T <---
6142024-02-27T21:44:11.544Z 25 500-519 2 2 2 4 4 4 F F F
6152024-02-27T21:44:11.544Z 26 520-539 1 1 1 1 1 1 F F F
6162024-02-27T21:44:11.544Z 27 540-559 1 1 1 1 1 1 F F F
6172024-02-27T21:44:11.544Z 28 560-579 4 4 1 8 8 1 F F T <---
6182024-02-27T21:44:11.544Z 29 580-599 3 3 3 5 5 5 F F F
6192024-02-27T21:44:11.544ZMax gen: 4, Max flush: 8
6202024-02-27T21:44:11.544ZError: Difference in extent metadata found!
6212024-02-27T21:44:11.544ZA Difference in extent metadata is expected here
6222024-02-27T21:44:11.544ZOn loop 2
6232024-02-27T21:44:11.544Z
6242024-02-27T21:44:11.552ZVerifying data now
6252024-02-27T21:44:17.121ZNo change after verify
6262024-02-27T21:44:17.121ZLoop: 2 Downstairs dump after verify (and repair):
6272024-02-27T21:44:17.202ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6282024-02-27T21:44:17.202Z 0 000-019 2 2 2 3 3 3 F F F
6292024-02-27T21:44:17.202Z 1 020-039 4 4 4 8 8 8 F F F
6302024-02-27T21:44:17.203Z 2 040-059 5 5 5 9 9 9 F F F
6312024-02-27T21:44:17.203Z 3 060-079 1 1 1 1 1 1 F F F
6322024-02-27T21:44:17.203Z 4 080-099 1 1 1 1 1 1 F F F
6332024-02-27T21:44:17.203Z 5 100-119 4 4 4 6 6 6 F F F
6342024-02-27T21:44:17.203Z 6 120-139 4 4 4 8 8 8 F F F
6352024-02-27T21:44:17.203Z 7 140-159 4 4 4 8 8 8 F F F
6362024-02-27T21:44:17.204Z 8 160-179 1 1 1 1 1 1 F F F
6372024-02-27T21:44:17.204Z 9 180-199 2 2 2 3 3 3 F F F
6382024-02-27T21:44:17.204Z 10 200-219 5 5 5 9 9 9 F F F
6392024-02-27T21:44:17.204Z 11 220-239 2 2 2 4 4 4 F F F
6402024-02-27T21:44:17.204Z 12 240-259 1 1 1 1 1 1 F F F
6412024-02-27T21:44:17.204Z 13 260-279 4 4 4 8 8 8 F F F
6422024-02-27T21:44:17.205Z 14 280-299 5 5 5 9 9 9 F F F
6432024-02-27T21:44:17.205Z 15 300-319 4 4 4 8 8 8 F F F
6442024-02-27T21:44:17.205Z 16 320-339 4 4 4 7 7 7 F F F
6452024-02-27T21:44:17.206Z 17 340-359 4 4 4 7 7 7 F F F
6462024-02-27T21:44:17.206Z 18 360-379 2 2 2 3 3 3 F F F
6472024-02-27T21:44:17.206Z 19 380-399 2 2 2 4 4 4 F F F
6482024-02-27T21:44:17.206Z 20 400-419 3 3 3 5 5 5 F F F
6492024-02-27T21:44:17.207Z 21 420-439 1 1 1 1 1 1 F F F
6502024-02-27T21:44:17.207Z 22 440-459 2 2 2 3 3 3 F F F
6512024-02-27T21:44:17.207Z 23 460-479 4 4 4 6 6 6 F F F
6522024-02-27T21:44:17.207Z 24 480-499 4 4 4 8 8 8 F F F
6532024-02-27T21:44:17.207Z 25 500-519 2 2 2 4 4 4 F F F
6542024-02-27T21:44:17.207Z 26 520-539 1 1 1 1 1 1 F F F
6552024-02-27T21:44:17.207Z 27 540-559 1 1 1 1 1 1 F F F
6562024-02-27T21:44:17.207Z 28 560-579 4 4 4 8 8 8 F F F
6572024-02-27T21:44:17.207Z 29 580-599 3 3 3 5 5 5 F F F
6582024-02-27T21:44:17.207ZMax gen: 5, Max flush: 9
6592024-02-27T21:44:17.207Z
6602024-02-27T21:44:17.207ZBegin loop 3
6612024-02-27T21:44:17.208ZDownstairs to restart: 2
6622024-02-27T21:44:17.208Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495082002Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162}
6632024-02-27T21:44:17.208Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.49535192Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162}
6642024-02-27T21:44:17.208Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495377967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162}
6652024-02-27T21:44:17.208Z{"msg":"Crucible runtime is spawned
6662024-02-27T21:44:17.208ZCrucible fac8711d-4c90-4887-86a5-a5277a779eec has session id: 56bddf85-777c-430d-be6d-00a87236d784","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495409113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6672024-02-27T21:44:17.208Z{"msg":"Upstairs opts: Upstairs UUID: fac8711d-4c90-4887-86a5-a5277a779eec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495455868Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6682024-02-27T21:44:17.208Z{"msg":"Crucible stats registered with UUID: fac8711d-4c90-4887-86a5-a5277a779eec","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495493763Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6692024-02-27T21:44:17.209Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495513701Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162}
6702024-02-27T21:44:17.209Z{"msg":"fac8711d-4c90-4887-86a5-a5277a779eec active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495633837Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6712024-02-27T21:44:17.209Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495670433Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6722024-02-27T21:44:17.209Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.49569506Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6732024-02-27T21:44:17.209Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495733125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6742024-02-27T21:44:17.209Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495764862Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6752024-02-27T21:44:17.209Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495800497Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6762024-02-27T21:44:17.209Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495823475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6772024-02-27T21:44:17.209Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495857251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"io task","client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6782024-02-27T21:44:17.210Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495892436Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"io task","client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6792024-02-27T21:44:17.210Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495921113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"io task","client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6802024-02-27T21:44:17.210Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495970477Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"io task","client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6812024-02-27T21:44:17.210Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.495996864Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"io task","client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6822024-02-27T21:44:17.210Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.496023831Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"io task","client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6832024-02-27T21:44:17.210Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.496808029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6842024-02-27T21:44:17.210Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.496847184Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6852024-02-27T21:44:17.210Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.496998986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6862024-02-27T21:44:17.211Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.497068548Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6872024-02-27T21:44:17.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.497210771Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6882024-02-27T21:44:17.211Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.497252586Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6892024-02-27T21:44:17.211Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.49764266Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6902024-02-27T21:44:17.211Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.497785673Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6912024-02-27T21:44:17.211Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.497977261Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6922024-02-27T21:44:17.211Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498231351Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6932024-02-27T21:44:17.212Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498279835Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6942024-02-27T21:44:17.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498300783Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6952024-02-27T21:44:17.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498314921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6962024-02-27T21:44:17.212Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498348357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6972024-02-27T21:44:17.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498385473Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6982024-02-27T21:44:17.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498483181Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
6992024-02-27T21:44:17.212Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498537465Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7002024-02-27T21:44:17.212Z{"msg":"[0]R flush_numbers[0..12]: [3, 8, 9, 1, 1, 6, 8, 8, 1, 3, 9, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498564832Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7012024-02-27T21:44:17.213Z{"msg":"[0]R generation[0..12]: [2, 4, 5, 1, 1, 4, 4, 4, 1, 2, 5, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.49858039Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7022024-02-27T21:44:17.213Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498602857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7032024-02-27T21:44:17.213Z{"msg":"[1]R flush_numbers[0..12]: [3, 8, 9, 1, 1, 6, 8, 8, 1, 3, 9, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498618146Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7042024-02-27T21:44:17.213Z{"msg":"[1]R generation[0..12]: [2, 4, 5, 1, 1, 4, 4, 4, 1, 2, 5, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498638953Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7052024-02-27T21:44:17.213Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498656631Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7062024-02-27T21:44:17.213Z{"msg":"[2]R flush_numbers[0..12]: [3, 8, 9, 1, 1, 6, 8, 8, 1, 3, 9, 4]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
7072024-02-27T21:44:17.213Z,"time":"2024-02-27T21:44:15.498691567Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7082024-02-27T21:44:17.213Z{"msg":"[2]R generation[0..12]: [2, 4, 5, 1, 1, 4, 4, 4, 1, 2, 5, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498732342Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7092024-02-27T21:44:17.214Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498765268Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7102024-02-27T21:44:17.214Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498809663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7112024-02-27T21:44:17.214Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498844069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7122024-02-27T21:44:17.214Z{"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498867686Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7132024-02-27T21:44:17.214Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498902472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"":"downstairs","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7142024-02-27T21:44:17.214Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498927429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7152024-02-27T21:44:17.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498940697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"0","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7162024-02-27T21:44:17.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498961005Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"1","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7172024-02-27T21:44:17.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.498997471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"client":"2","":"downstairs","session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7182024-02-27T21:44:17.215Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.499023098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7192024-02-27T21:44:17.215Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.499045485Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7202024-02-27T21:44:17.215Z{"msg":"fac8711d-4c90-4887-86a5-a5277a779eec is now active with session: 56bddf85-777c-430d-be6d-00a87236d784","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.499059874Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7212024-02-27T21:44:17.216Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.49909102Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162,"session_id":"56bddf85-777c-430d-be6d-00a87236d784"}
7222024-02-27T21:44:17.216Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:15.499113967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162}
7232024-02-27T21:44:17.216ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7242024-02-27T21:44:17.216ZLoading write count information from file "/tmp/test_repair_verify.data"
7252024-02-27T21:44:17.216ZRun Repair workload
7262024-02-27T21:44:17.216Z01/30 Write block 573 len 4096 data: 3
7272024-02-27T21:44:17.688Z02/30 Write block 87 len 16384 data: 2 2 2 2
7282024-02-27T21:44:17.688Z03/30 Read block 81 len 8192
7292024-02-27T21:44:17.688Z04/30 Flush
7302024-02-27T21:44:17.688Z05/30 Write block 216 len 28672 data: 2 2 2 2 2 2 2
7312024-02-27T21:44:17.689Z06/30 Read block 445 len 28672
7322024-02-27T21:44:17.689Z07/30 Read block 129 len 12288
7332024-02-27T21:44:17.689Z08/30 Flush
7342024-02-27T21:44:17.689Z09/30 Write block 172 len 16384 data: 2 2 2 2
7352024-02-27T21:44:17.689Z10/30 Write block 497 len 12288 data: 3 3 3
7362024-02-27T21:44:17.689Z11/30 Read block 502 len 20480
7372024-02-27T21:44:17.690Z12/30 Flush
7382024-02-27T21:44:17.690Z13/30 Write block 168 len 12288 data: 2 2 2
7392024-02-27T21:44:17.690Z14/30 Write block 227 len 28672 data: 3 2 2 2 2 2 2
7402024-02-27T21:44:17.690Z15/30 Read block 39 len 36864
7412024-02-27T21:44:17.690Z16/30 Read block 90 len 4096
7422024-02-27T21:44:17.691Z17/30 Flush
7432024-02-27T21:44:17.691Z18/30 Write block 88 len 8192 data: 3 3
7442024-02-27T21:44:17.691Z19/30 Write block 446 len 16384 data: 3 3 3 2
7452024-02-27T21:44:17.691Z20/30 Flush
7462024-02-27T21:44:17.691Z21/30 Write block 190 len 8192 data: 3 3
7472024-02-27T21:44:17.692Z22/30 Read block 192 len 8192
7482024-02-27T21:44:17.692Z23/30 Write block 575 len 24576 data: 3 3 3 3 2 2
7492024-02-27T21:44:17.692Z24/30 Read block 80 len 32768
7502024-02-27T21:44:17.692Z25/30 Read block 143 len 16384
7512024-02-27T21:44:17.692Z26/30 Read block 12 len 40960
7522024-02-27T21:44:17.692Z27/30 Read block 428 len 20480
7532024-02-27T21:44:17.692Z28/30 Read block 27 len 32768
7542024-02-27T21:44:17.692Z29/30 Read block 328 len 8192
7552024-02-27T21:44:17.692Z30/30 Write block 574 len 24576 data: 3 4 4 4 4 3
7562024-02-27T21:44:17.693Z----------------------------------------------------------------
7572024-02-27T21:44:17.693Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7582024-02-27T21:44:17.693ZStates: Active Active Active
7592024-02-27T21:44:17.693ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7602024-02-27T21:44:17.693Z 1 Acked 1000 Write 1 Done Done Done false
7612024-02-27T21:44:17.693Z 2 Acked 1001 Write 4 Done Done Sent false
7622024-02-27T21:44:17.693Z 3 Acked 1002 Read 2 Done Done Sent false
7632024-02-27T21:44:17.693Z 4 Acked 1003 Flush 0 Done Done Sent false
7642024-02-27T21:44:17.693Z 5 Acked 1004 Write 7 Done Done Sent false
7652024-02-27T21:44:17.694Z 6 Acked 1005 Read 7 Done Done Sent false
7662024-02-27T21:44:17.694Z 7 Acked 1006 Read 3 Done Done Sent false
7672024-02-27T21:44:17.694Z 8 Acked 1007 Flush 0 Done Done Sent false
7682024-02-27T21:44:17.694Z 9 Acked 1008 Write 4 Done Done Sent false
7692024-02-27T21:44:17.694Z 10 Acked 1009 Write 3 Done Done Sent false
7702024-02-27T21:44:17.694Z 11 Acked 1010 Read 5 Done Done Sent false
7712024-02-27T21:44:17.694Z 12 Acked 1011 Flush 0 Done Done Sent false
7722024-02-27T21:44:17.694Z 13 Acked 1012 Write 3 Done Done Sent false
7732024-02-27T21:44:17.694Z 14 Acked 1013 Write 7 Done Done Sent false
7742024-02-27T21:44:17.695Z 15 Acked 1014 Read 9 Done Done Sent false
7752024-02-27T21:44:17.695Z 16 Acked 1015 Read 1 Done Done Sent false
7762024-02-27T21:44:17.695Z 17 Acked 1016 Flush 0 Done Done Sent false
7772024-02-27T21:44:17.695Z 18 Acked 1017 Write 2 Done Done Sent false
7782024-02-27T21:44:17.695Z 19 Acked 1018 Write 4 Done Done Sent false
7792024-02-27T21:44:17.696Z 20 Acked 1019 Flush 0 Done Done Sent false
7802024-02-27T21:44:17.696Z 21 Acked 1020 Write 2 Done Done Sent false
7812024-02-27T21:44:17.696Z 22 Acked 1021 Read 2 Done Done Sent false
7822024-02-27T21:44:17.696Z 23 Acked 1022 Write 6 Done Done Sent false
7832024-02-27T21:44:17.696Z 24 Acked 1023 Read 8 Done Done Sent false
7842024-02-27T21:44:17.697Z 25 Acked 1024 Read 4 Done Done Sent false
7852024-02-27T21:44:17.697Z 26 Acked 1025 Read 10 Done Done Sent false
7862024-02-27T21:44:17.697Z 27 Acked 1026 Read 5 Done Done Sent false
7872024-02-27T21:44:17.697Z 28 Acked 1027 Read 8 Done Done Sent false
7882024-02-27T21:44:17.697Z 29 Acked 1028 Read 2 Done Done Sent false
7892024-02-27T21:44:17.697Z 30 Acked 1029 Write 6 Sent Sent Sent false
7902024-02-27T21:44:17.697Z STATES DS:0 DS:1 DS:2 TOTAL
7912024-02-27T21:44:17.697Z New 0 0 0 0
7922024-02-27T21:44:17.697Z Sent 1 1 29 31
7932024-02-27T21:44:17.698Z Done 29 29 1 59
7942024-02-27T21:44:17.698Z Skipped 0 0 0 0
7952024-02-27T21:44:17.698Z Error 0 0 0 0
7962024-02-27T21:44:17.698ZLast Flush: 1019 1019 0
7972024-02-27T21:44:17.698ZDownstairs last five completed:
7982024-02-27T21:44:17.699ZUpstairs last five completed: 30 29 28 27 26
7992024-02-27T21:44:17.699Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:15.529972332Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1162}
8002024-02-27T21:44:17.699ZWrote out file "/tmp/test_repair_verify.data"
8012024-02-27T21:44:17.699Z
8022024-02-27T21:44:17.699ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8032024-02-27T21:44:17.699ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8042024-02-27T21:44:17.700Z 0 000-019 2 2 2 3 3 3 F F F
8052024-02-27T21:44:17.700Z 1 020-039 4 4 4 8 8 8 F F F
8062024-02-27T21:44:17.700Z 2 040-059 5 5 5 9 9 9 F F F
8072024-02-27T21:44:17.700Z 3 060-079 1 1 1 1 1 1 F F F
8082024-02-27T21:44:17.700Z 4 080-099 6 6 1 14 14 1 F F F <---
8092024-02-27T21:44:17.700Z 5 100-119 4 4 4 6 6 6 F F F
8102024-02-27T21:44:17.700Z 6 120-139 4 4 4 8 8 8 F F F
8112024-02-27T21:44:17.700Z 7 140-159 4 4 4 8 8 8 F F F
8122024-02-27T21:44:17.700Z 8 160-179 6 6 1 13 13 1 F F F <---
8132024-02-27T21:44:17.701Z 9 180-199 2 2 2 3 3 3 T T F <---
8142024-02-27T21:44:17.701Z 10 200-219 6 6 5 11 11 9 F F F <---
8152024-02-27T21:44:17.701Z 11 220-239 6 6 2 13 13 4 F F F <---
8162024-02-27T21:44:17.701Z 12 240-259 1 1 1 1 1 1 F F F
8172024-02-27T21:44:17.701Z 13 260-279 4 4 4 8 8 8 F F F
8182024-02-27T21:44:17.702Z 14 280-299 5 5 5 9 9 9 F F F
8192024-02-27T21:44:17.702Z 15 300-319 4 4 4 8 8 8 F F F
8202024-02-27T21:44:17.702Z 16 320-339 4 4 4 7 7 7 F F F
8212024-02-27T21:44:17.702Z 17 340-359 4 4 4 7 7 7 F F F
8222024-02-27T21:44:17.702Z 18 360-379 2 2 2 3 3 3 F F F
8232024-02-27T21:44:17.702Z 19 380-399 2 2 2 4 4 4 F F F
8242024-02-27T21:44:17.702Z 20 400-419 3 3 3 5 5 5 F F F
8252024-02-27T21:44:17.702Z 21 420-439 1 1 1 1 1 1 F F F
8262024-02-27T21:44:17.703Z 22 440-459 6 6 2 14 14 3 F F F <---
8272024-02-27T21:44:17.720Z 23 460-479 4 4 4 6 6 6 F F F
8282024-02-27T21:44:17.720Z 24 480-499 6 6 4 12 12 8 F F F <---
8292024-02-27T21:44:17.720Z 25 500-519 2 2 2 4 4 4 F F F
8302024-02-27T21:44:17.721Z 26 520-539 1 1 1 1 1 1 F F F
8312024-02-27T21:44:17.721Z 27 540-559 1 1 1 1 1 1 F F F
8322024-02-27T21:44:17.721Z 28 560-579 6 6 4 10 10 8 T T T <---
8332024-02-27T21:44:17.721Z 29 580-599 3 3 3 5 5 5 T T F <---
8342024-02-27T21:44:17.721ZMax gen: 6, Max flush: 14
8352024-02-27T21:44:17.721ZError: Difference in extent metadata found!
8362024-02-27T21:44:17.721ZA Difference in extent metadata is expected here
8372024-02-27T21:44:17.721ZOn loop 3
8382024-02-27T21:44:17.721Z
8392024-02-27T21:44:17.721ZVerifying data now
8402024-02-27T21:44:21.637ZNo change after verify
8412024-02-27T21:44:21.637ZLoop: 3 Downstairs dump after verify (and repair):
8422024-02-27T21:44:21.684ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8432024-02-27T21:44:21.684Z 0 000-019 2 2 2 3 3 3 F F F
8442024-02-27T21:44:21.684Z 1 020-039 4 4 4 8 8 8 F F F
8452024-02-27T21:44:21.684Z 2 040-059 5 5 5 9 9 9 F F F
8462024-02-27T21:44:21.685Z 3 060-079 1 1 1 1 1 1 F F F
8472024-02-27T21:44:21.685Z 4 080-099 6 6 6 14 14 14 F F F
8482024-02-27T21:44:21.685Z 5 100-119 4 4 4 6 6 6 F F F
8492024-02-27T21:44:21.685Z 6 120-139 4 4 4 8 8 8 F F F
8502024-02-27T21:44:21.685Z 7 140-159 4 4 4 8 8 8 F F F
8512024-02-27T21:44:21.685Z 8 160-179 6 6 6 13 13 13 F F F
8522024-02-27T21:44:21.686Z 9 180-199 7 7 7 15 15 15 F F F
8532024-02-27T21:44:21.686Z 10 200-219 6 6 6 11 11 11 F F F
8542024-02-27T21:44:21.686Z 11 220-239 6 6 6 13 13 13 F F F
8552024-02-27T21:44:21.686Z 12 240-259 1 1 1 1 1 1 F F F
8562024-02-27T21:44:21.686Z 13 260-279 4 4 4 8 8 8 F F F
8572024-02-27T21:44:21.686Z 14 280-299 5 5 5 9 9 9 F F F
8582024-02-27T21:44:21.687Z 15 300-319 4 4 4 8 8 8 F F F
8592024-02-27T21:44:21.687Z 16 320-339 4 4 4 7 7 7 F F F
8602024-02-27T21:44:21.687Z 17 340-359 4 4 4 7 7 7 F F F
8612024-02-27T21:44:21.687Z 18 360-379 2 2 2 3 3 3 F F F
8622024-02-27T21:44:21.688Z 19 380-399 2 2 2 4 4 4 F F F
8632024-02-27T21:44:21.688Z 20 400-419 3 3 3 5 5 5 F F F
8642024-02-27T21:44:21.688Z 21 420-439 1 1 1 1 1 1 F F F
8652024-02-27T21:44:21.688Z 22 440-459 6 6 6 14 14 14 F F F
8662024-02-27T21:44:21.688Z 23 460-479 4 4 4 6 6 6 F F F
8672024-02-27T21:44:21.688Z 24 480-499 6 6 6 12 12 12 F F F
8682024-02-27T21:44:21.689Z 25 500-519 2 2 2 4 4 4 F F F
8692024-02-27T21:44:21.689Z 26 520-539 1 1 1 1 1 1 F F F
8702024-02-27T21:44:21.689Z 27 540-559 1 1 1 1 1 1 F F F
8712024-02-27T21:44:21.689Z 28 560-579 7 7 7 15 15 15 F F F
8722024-02-27T21:44:21.689Z 29 580-599 7 7 7 15 15 15 F F F
8732024-02-27T21:44:21.689ZMax gen: 7, Max flush: 15
8742024-02-27T21:44:21.690Z
8752024-02-27T21:44:21.690ZBegin loop 4
8762024-02-27T21:44:21.690ZDownstairs to restart: 1
8772024-02-27T21:44:21.690Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058332667Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170}
8782024-02-27T21:44:21.690Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058606004Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170}
8792024-02-27T21:44:21.690Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058624142Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170}
8802024-02-27T21:44:21.690Z{"msg":"Crucible 76825b27-f04f-4cca-80d5-10119df61e33 has session id: 8f71bd7f-3bbd-424c-99c9-5e9a56a367fe","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.05864571Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8812024-02-27T21:44:21.691Z{"Crucible runtime is spawned
8822024-02-27T21:44:21.691Zmsg":"Upstairs opts: Upstairs UUID: 76825b27-f04f-4cca-80d5-10119df61e33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058702483Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8832024-02-27T21:44:21.691Z{"msg":"Crucible stats registered with UUID: 76825b27-f04f-4cca-80d5-10119df61e33","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058735599Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8842024-02-27T21:44:21.691Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058758646Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170}
8852024-02-27T21:44:21.691Z{"msg":"76825b27-f04f-4cca-80d5-10119df61e33 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058792153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8862024-02-27T21:44:21.691Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058807211Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8872024-02-27T21:44:21.692Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058840627Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8882024-02-27T21:44:21.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058868884Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8892024-02-27T21:44:21.692Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058905339Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8902024-02-27T21:44:21.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058933246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8912024-02-27T21:44:21.693Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.058970761Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8922024-02-27T21:44:21.693Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.059033284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"io task","client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8932024-02-27T21:44:21.693Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.059094117Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"io task","client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8942024-02-27T21:44:21.693Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.059124083Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"io task","client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8952024-02-27T21:44:21.693Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.059250968Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"io task","client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8962024-02-27T21:44:21.693Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.059313481Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"io task","client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8972024-02-27T21:44:21.694Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.059362725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"io task","client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8982024-02-27T21:44:21.694Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.06000697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
8992024-02-27T21:44:21.694Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.060053634Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9002024-02-27T21:44:21.694Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.060176959Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9012024-02-27T21:44:21.694Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.060219774Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9022024-02-27T21:44:21.695Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.060312063Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9032024-02-27T21:44:21.695Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.060366967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9042024-02-27T21:44:21.695Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.060536667Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9052024-02-27T21:44:21.695Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.060871857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9062024-02-27T21:44:21.695Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061069864Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9072024-02-27T21:44:21.695Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.06111042Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9082024-02-27T21:44:21.696Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061126427Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9092024-02-27T21:44:21.696Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061140256Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9102024-02-27T21:44:21.696Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061333803Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9112024-02-27T21:44:21.696Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061369729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9122024-02-27T21:44:21.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061400135Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9132024-02-27T21:44:21.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061577834Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9142024-02-27T21:44:21.703Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061619969Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9152024-02-27T21:44:21.703Z{"msg":"[0]R flush_numbers[0..12]: [3, 8, 9, 1, 14, 6, 8, 8, 13, 15, 11, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061649366Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9162024-02-27T21:44:21.703Z{"msg":"[0]R generation[0..12]: [2, 4, 5, 1, 6, 4, 4, 4, 6, 7, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061685122Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9172024-02-27T21:44:21.704ZWait for a query_work_queue command to finish before sending IO
9182024-02-27T21:44:21.704Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061728357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9192024-02-27T21:44:21.704Z{"msg":"[1]R flush_numbers[0..12]: [3, 8, 9, 1, 14, 6, 8, 8, 13, 15, 11, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061761233Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9202024-02-27T21:44:21.704Z{"msg":"[1]R generation[0..12]: [2, 4, 5, 1, 6, 4, 4, 4, 6, 7, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.06178756Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9212024-02-27T21:44:21.704Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061818686Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9222024-02-27T21:44:21.704Z{"msg":"[2]R flush_numbers[0..12]: [3, 8, 9, 1, 14, 6, 8, 8, 13, 15, 11, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061845403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9232024-02-27T21:44:21.704Z{"msg":"[2]R generation[0..12]: [2, 4, 5, 1, 6, 4, 4, 4, 6, 7, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.06187503Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9242024-02-27T21:44:21.705Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061901876Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9252024-02-27T21:44:21.705Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061930663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9262024-02-27T21:44:21.705Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.0619552Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9272024-02-27T21:44:21.705Z{"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.061981217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9282024-02-27T21:44:21.705Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.062005054Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"":"downstairs","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9292024-02-27T21:44:21.705Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.062030071Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9302024-02-27T21:44:21.706Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.062043439Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"0","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9312024-02-27T21:44:21.706Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.062056998Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"1","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9322024-02-27T21:44:21.706Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.062070966Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"client":"2","":"downstairs","session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9332024-02-27T21:44:21.706Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.062092104Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9342024-02-27T21:44:21.706Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9352024-02-27T21:44:21.706Z2024-02-27T21:44:20.06212672Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9362024-02-27T21:44:21.706Z{"msg":"76825b27-f04f-4cca-80d5-10119df61e33 is now active with session: 8f71bd7f-3bbd-424c-99c9-5e9a56a367fe","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.062157696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9372024-02-27T21:44:21.706Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.062177944Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170,"session_id":"8f71bd7f-3bbd-424c-99c9-5e9a56a367fe"}
9382024-02-27T21:44:21.707Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:20.062190682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170}
9392024-02-27T21:44:22.840ZLoading write count information from file "/tmp/test_repair_verify.data"
9402024-02-27T21:44:22.840ZRun Repair workload
9412024-02-27T21:44:22.840Z01/30 Write block 359 len 40960 data: 3 3 2 2 2 2 2 2 2 2
9422024-02-27T21:44:22.840Z02/30 Read block 42 len 36864
9432024-02-27T21:44:22.840Z03/30 Write block 117 len 24576 data: 3 3 3 3 3 3
9442024-02-27T21:44:22.840Z04/30 Write block 245 len 12288 data: 2 2 2
9452024-02-27T21:44:22.840Z05/30 Read block 263 len 36864
9462024-02-27T21:44:22.840Z06/30 Read block 428 len 40960
9472024-02-27T21:44:22.841Z07/30 Read block 179 len 36864
9482024-02-27T21:44:22.841Z08/30 Write block 188 len 8192 data: 2 3
9492024-02-27T21:44:22.841Z09/30 Write block 217 len 8192 data: 3 3
9502024-02-27T21:44:22.841Z10/30 Read block 278 len 40960
9512024-02-27T21:44:22.841Z11/30 Write block 385 len 28672 data: 2 2 4 4 4 3 3
9522024-02-27T21:44:22.841Z12/30 Read block 437 len 12288
9532024-02-27T21:44:22.841Z13/30 Write block 39 len 24576 data: 3 3 3 3 4 4
9542024-02-27T21:44:22.841Z14/30 Read block 424 len 40960
9552024-02-27T21:44:22.841Z15/30 Write block 169 len 24576 data: 3 3 2 3 3 3
9562024-02-27T21:44:22.841Z16/30 Read block 483 len 24576
9572024-02-27T21:44:22.841Z17/30 Read block 5 len 32768
9582024-02-27T21:44:22.841Z18/30 Write block 227 len 36864 data: 4 3 3 3 3 3 3 2 2
9592024-02-27T21:44:22.841Z19/30 Read block 48 len 32768
9602024-02-27T21:44:22.841Z20/30 Read block 574 len 16384
9612024-02-27T21:44:22.841Z21/30 Read block 30 len 12288
9622024-02-27T21:44:22.841Z22/30 Write block 548 len 40960 data: 2 2 2 2 2 2 2 2 2 2
9632024-02-27T21:44:22.841Z23/30 Write block 463 len 4096 data: 2
9642024-02-27T21:44:22.841Z24/30 Read block 584 len 28672
9652024-02-27T21:44:22.842Z25/30 Write block 401 len 16384 data: 2 2 2 2
9662024-02-27T21:44:22.842Z26/30 Read block 92 len 32768
9672024-02-27T21:44:22.842Z27/30 Read block 246 len 28672
9682024-02-27T21:44:22.842Z28/30 Write block 77 len 12288 data: 2 2 2
9692024-02-27T21:44:22.842Z29/30 Read block 329 len 4096
9702024-02-27T21:44:22.842Z30/30 Read block 456 len 12288
9712024-02-27T21:44:22.842Z----------------------------------------------------------------
9722024-02-27T21:44:22.842Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9732024-02-27T21:44:22.842ZStates: Active Active Active
9742024-02-27T21:44:22.842ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9752024-02-27T21:44:22.842Z 1 Acked 1000 Write 10 Done Done Done false
9762024-02-27T21:44:22.842Z 2 Acked 1001 Read 9 Done Done Done false
9772024-02-27T21:44:22.842Z 3 Acked 1002 Write 6 Done Sent Done false
9782024-02-27T21:44:22.842Z 4 Acked 1003 Write 3 Done Sent Done false
9792024-02-27T21:44:22.842Z 5 Acked 1004 Read 9 Done Sent Done false
9802024-02-27T21:44:22.843Z 6 Acked 1005 Read 10 Done Sent Done false
9812024-02-27T21:44:22.843Z 7 Acked 1006 Read 9 Done Sent Done false
9822024-02-27T21:44:22.843Z 8 Acked 1007 Write 2 Done Sent Done false
9832024-02-27T21:44:22.843Z 9 Acked 1008 Write 2 Done Sent Done false
9842024-02-27T21:44:22.843Z 10 Acked 1009 Read 10 Done Sent Done false
9852024-02-27T21:44:22.843Z 11 Acked 1010 Write 7 Done Sent Done false
9862024-02-27T21:44:22.843Z 12 Acked 1011 Read 3 Done Sent Done false
9872024-02-27T21:44:22.843Z 13 Acked 1012 Write 6 Done Sent Done false
9882024-02-27T21:44:22.843Z 14 Acked 1013 Read 10 Done Sent Done false
9892024-02-27T21:44:22.843Z 15 Acked 1014 Write 6 Done Sent Done false
9902024-02-27T21:44:22.843Z 16 Acked 1015 Read 6 Done Sent Done false
9912024-02-27T21:44:22.843Z 17 Acked 1016 Read 8 Done Sent Done false
9922024-02-27T21:44:22.844Z 18 Acked 1017 Write 9 Done Sent Done false
9932024-02-27T21:44:22.844Z 19 Acked 1018 Read 8 Done Sent Done false
9942024-02-27T21:44:22.844Z 20 Acked 1019 Read 4 Done Sent Done false
9952024-02-27T21:44:22.844Z 21 Acked 1020 Read 3 Done Sent Done false
9962024-02-27T21:44:22.844Z 22 Acked 1021 Write 10 Done Sent Done false
9972024-02-27T21:44:22.844Z 23 Acked 1022 Write 1 Done Sent Done false
9982024-02-27T21:44:22.844Z 24 Acked 1023 Read 7 Done Sent Done false
9992024-02-27T21:44:22.844Z 25 Acked 1024 Write 4 Done Sent Done false
10002024-02-27T21:44:22.844Z 26 Acked 1025 Read 8 Done Sent Done false
10012024-02-27T21:44:22.844Z 27 Acked 1026 Read 7 Done Sent Done false
10022024-02-27T21:44:22.844Z 28 Acked 1027 Write 3 Done Sent Done false
10032024-02-27T21:44:22.844Z 29 Acked 1028 Read 1 Done Sent Done false
10042024-02-27T21:44:22.844Z 30 Acked 1029 Read 3 Done Sent Done false
10052024-02-27T21:44:22.845Z STATES DS:0 DS:1 DS:2 TOTAL
10062024-02-27T21:44:22.845Z New 0 0 0 0
10072024-02-27T21:44:22.845Z Sent 0 28 0 28
10082024-02-27T21:44:22.845Z Done 30 2 30 62
10092024-02-27T21:44:22.845Z Skipped 0 0 0 0
10102024-02-27T21:44:22.845Z Error 0 0 0 0
10112024-02-27T21:44:22.845ZLast Flush: 0 0 0
10122024-02-27T21:44:22.845ZDownstairs last five completed:
10132024-02-27T21:44:22.845ZUpstairs last five completed: 30 29 28 27 26
10142024-02-27T21:44:22.845Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:20.091474083Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1170}
10152024-02-27T21:44:22.845ZWrote out file "/tmp/test_repair_verify.data"
10162024-02-27T21:44:22.846Z
10172024-02-27T21:44:22.846ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10182024-02-27T21:44:22.846ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10192024-02-27T21:44:22.846Z 0 000-019 2 2 2 3 3 3 F F F
10202024-02-27T21:44:22.846Z 1 020-039 4 4 4 8 8 8 T F T <---
10212024-02-27T21:44:22.846Z 2 040-059 5 5 5 9 9 9 T F T <---
10222024-02-27T21:44:22.846Z 3 060-079 1 1 1 1 1 1 T F T <---
10232024-02-27T21:44:22.846Z 4 080-099 6 6 6 14 14 14 F F F
10242024-02-27T21:44:22.846Z 5 100-119 4 4 4 6 6 6 T F T <---
10252024-02-27T21:44:22.846Z 6 120-139 4 4 4 8 8 8 T F T <---
10262024-02-27T21:44:22.847Z 7 140-159 4 4 4 8 8 8 F F F
10272024-02-27T21:44:22.847Z 8 160-179 6 6 6 13 13 13 T F T <---
10282024-02-27T21:44:22.847Z 9 180-199 7 7 7 15 15 15 T F T <---
10292024-02-27T21:44:22.847Z 10 200-219 6 6 6 11 11 11 T F T <---
10302024-02-27T21:44:22.847Z 11 220-239 6 6 6 13 13 13 T F T <---
10312024-02-27T21:44:22.847Z 12 240-259 1 1 1 1 1 1 T F T <---
10322024-02-27T21:44:22.847Z 13 260-279 4 4 4 8 8 8 F F F
10332024-02-27T21:44:22.847Z 14 280-299 5 5 5 9 9 9 F F F
10342024-02-27T21:44:22.847Z 15 300-319 4 4 4 8 8 8 F F F
10352024-02-27T21:44:22.847Z 16 320-339 4 4 4 7 7 7 F F F
10362024-02-27T21:44:22.847Z 17 340-359 4 4 4 7 7 7 T T T
10372024-02-27T21:44:22.847Z 18 360-379 2 2 2 3 3 3 T T T
10382024-02-27T21:44:22.847Z 19 380-399 2 2 2 4 4 4 T F T <---
10392024-02-27T21:44:22.924Z 20 400-419 3 3 3 5 5 5 T F T <---
10402024-02-27T21:44:22.924Z 21 420-439 1 1 1 1 1 1 F F F
10412024-02-27T21:44:22.924Z 22 440-459 6 6 6 14 14 14 F F F
10422024-02-27T21:44:22.924Z 23 460-479 4 4 4 6 6 6 T F T <---
10432024-02-27T21:44:22.924Z 24 480-499 6 6 6 12 12 12 F F F
10442024-02-27T21:44:22.924Z 25 500-519 2 2 2 4 4 4 F F F
10452024-02-27T21:44:22.924Z 26 520-539 1 1 1 1 1 1 F F F
10462024-02-27T21:44:22.925Z 27 540-559 1 1 1 1 1 1 T F T <---
10472024-02-27T21:44:22.925Z 28 560-579 7 7 7 15 15 15 F F F
10482024-02-27T21:44:22.925Z 29 580-599 7 7 7 15 15 15 F F F
10492024-02-27T21:44:22.925ZMax gen: 7, Max flush: 15
10502024-02-27T21:44:22.925ZError: Difference in extent metadata found!
10512024-02-27T21:44:22.925ZA Difference in extent metadata is expected here
10522024-02-27T21:44:22.926ZOn loop 4
10532024-02-27T21:44:22.926Z
10542024-02-27T21:44:22.926ZVerifying data now
10552024-02-27T21:44:27.492ZNo change after verify
10562024-02-27T21:44:27.492ZLoop: 4 Downstairs dump after verify (and repair):
10572024-02-27T21:44:27.509ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10582024-02-27T21:44:27.510Z 0 000-019 2 2 2 3 3 3 F F F
10592024-02-27T21:44:27.510Z 1 020-039 8 8 8 16 16 16 F F F
10602024-02-27T21:44:27.510Z 2 040-059 8 8 8 16 16 16 F F F
10612024-02-27T21:44:27.510Z 3 060-079 8 8 8 16 16 16 F F F
10622024-02-27T21:44:27.510Z 4 080-099 6 6 6 14 14 14 F F F
10632024-02-27T21:44:27.510Z 5 100-119 8 8 8 16 16 16 F F F
10642024-02-27T21:44:27.520Z 6 120-139 8 8 8 16 16 16 F F F
10652024-02-27T21:44:27.520Z 7 140-159 4 4 4 8 8 8 F F F
10662024-02-27T21:44:27.520Z 8 160-179 8 8 8 16 16 16 F F F
10672024-02-27T21:44:27.521Z 9 180-199 8 8 8 16 16 16 F F F
10682024-02-27T21:44:27.521Z 10 200-219 8 8 8 16 16 16 F F F
10692024-02-27T21:44:27.521Z 11 220-239 8 8 8 16 16 16 F F F
10702024-02-27T21:44:27.521Z 12 240-259 8 8 8 16 16 16 F F F
10712024-02-27T21:44:27.521Z 13 260-279 4 4 4 8 8 8 F F F
10722024-02-27T21:44:27.521Z 14 280-299 5 5 5 9 9 9 F F F
10732024-02-27T21:44:27.522Z 15 300-319 4 4 4 8 8 8 F F F
10742024-02-27T21:44:27.522Z 16 320-339 4 4 4 7 7 7 F F F
10752024-02-27T21:44:27.522Z 17 340-359 8 8 8 16 16 16 F F F
10762024-02-27T21:44:27.522Z 18 360-379 8 8 8 16 16 16 F F F
10772024-02-27T21:44:27.522Z 19 380-399 8 8 8 16 16 16 F F F
10782024-02-27T21:44:27.522Z 20 400-419 8 8 8 16 16 16 F F F
10792024-02-27T21:44:27.522Z 21 420-439 1 1 1 1 1 1 F F F
10802024-02-27T21:44:27.522Z 22 440-459 6 6 6 14 14 14 F F F
10812024-02-27T21:44:27.523Z 23 460-479 8 8 8 16 16 16 F F F
10822024-02-27T21:44:27.523Z 24 480-499 6 6 6 12 12 12 F F F
10832024-02-27T21:44:27.523Z 25 500-519 2 2 2 4 4 4 F F F
10842024-02-27T21:44:27.523Z 26 520-539 1 1 1 1 1 1 F F F
10852024-02-27T21:44:27.523Z 27 540-559 8 8 8 16 16 16 F F F
10862024-02-27T21:44:27.523Z 28 560-579 7 7 7 15 15 15 F F F
10872024-02-27T21:44:27.523Z 29 580-599 7 7 7 15 15 15 F F F
10882024-02-27T21:44:27.524ZMax gen: 8, Max flush: 16
10892024-02-27T21:44:27.524Z
10902024-02-27T21:44:27.524ZBegin loop 5
10912024-02-27T21:44:27.524ZDownstairs to restart: 0
10922024-02-27T21:44:27.548Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.937721871Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178}
10932024-02-27T21:44:27.548Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938142482Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178}
10942024-02-27T21:44:27.548Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
10952024-02-27T21:44:27.549Z,"time":"2024-02-27T21:44:25.938176158Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178}
10962024-02-27T21:44:27.549Z{"msg":"Crucible 79d20f19-33ae-49bc-9bf8-88e40928905f has session id: 7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938207914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
10972024-02-27T21:44:27.549Z{"msg":"Upstairs opts: Upstairs UUID: 79d20f19-33ae-49bc-9bf8-88e40928905f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938265007Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
10982024-02-27T21:44:27.550Z{"msg":"Crucible stats registered with UUID: 79d20f19-33ae-49bc-9bf8-88e40928905f","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938310252Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
10992024-02-27T21:44:27.550Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938339558Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178}
11002024-02-27T21:44:27.550Z{"msg":"79d20f19-33ae-49bc-9bf8-88e40928905f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938371595Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11012024-02-27T21:44:27.550Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.9384097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11022024-02-27T21:44:27.550Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938448026Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11032024-02-27T21:44:27.550Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938488081Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11042024-02-27T21:44:27.551Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938520867Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11052024-02-27T21:44:27.551Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938554223Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11062024-02-27T21:44:27.551Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.93858286Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11072024-02-27T21:44:27.551Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938611946Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"io task","client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11082024-02-27T21:44:27.551Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.93867079Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"io task","client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11092024-02-27T21:44:27.551Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938726573Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"io task","client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11102024-02-27T21:44:27.552Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938777507Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"io task","client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11112024-02-27T21:44:27.552Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938818712Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"io task","client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11122024-02-27T21:44:27.552Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.938909461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"io task","client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11132024-02-27T21:44:27.552Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.939466846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11142024-02-27T21:44:27.552Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.939504221Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11152024-02-27T21:44:27.552Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.939520949Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11162024-02-27T21:44:27.553Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.939537257Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11172024-02-27T21:44:27.553Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.939559895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11182024-02-27T21:44:27.553Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.939583772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11192024-02-27T21:44:27.553Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940098851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11202024-02-27T21:44:27.553Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940140056Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11212024-02-27T21:44:27.553Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940161024Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11222024-02-27T21:44:27.553Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940629439Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11232024-02-27T21:44:27.553Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940681333Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11242024-02-27T21:44:27.554Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940772142Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11252024-02-27T21:44:27.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940811117Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11262024-02-27T21:44:27.554Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940837634Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11272024-02-27T21:44:27.554Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940865251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11282024-02-27T21:44:27.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940888378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11292024-02-27T21:44:27.554Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940916235Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11302024-02-27T21:44:27.554Z{"msg":"[0]R flush_numbers[0..12]: [3, 16, 16, 16, 14, 16, 16, 8, 16, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940949011Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":Wait for a query_work_queue command to finish before sending IO
11312024-02-27T21:44:27.555Z"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11322024-02-27T21:44:27.555Z{"msg":"[0]R generation[0..12]: [2, 8, 8, 8, 6, 8, 8, 4, 8, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.940985357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11332024-02-27T21:44:27.555Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941020153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11342024-02-27T21:44:27.555Z{"msg":"[1]R flush_numbers[0..12]: [3, 16, 16, 16, 14, 16, 16, 8, 16, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941053139Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11352024-02-27T21:44:27.555Z{"msg":"[1]R generation[0..12]: [2, 8, 8, 8, 6, 8, 8, 4, 8, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941076766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11362024-02-27T21:44:27.555Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941105763Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11372024-02-27T21:44:27.555Z{"msg":"[2]R flush_numbers[0..12]: [3, 16, 16, 16, 14, 16, 16, 8, 16, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.94113192Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11382024-02-27T21:44:27.556Z{"msg":"[2]R generation[0..12]: [2, 8, 8, 8, 6, 8, 8, 4, 8, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941162916Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11392024-02-27T21:44:27.556Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941188943Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11402024-02-27T21:44:27.556Z{"msg":"Max found gen is 9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.94121508Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11412024-02-27T21:44:27.556Z{"msg":"Generation requested: 10 >= found:9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941238747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11422024-02-27T21:44:27.556Z{"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941264614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11432024-02-27T21:44:27.556Z,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11442024-02-27T21:44:27.556Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.94129588Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"":"downstairs","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11452024-02-27T21:44:27.556Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941325137Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11462024-02-27T21:44:27.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941347284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"0","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11472024-02-27T21:44:27.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941374411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"1","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11482024-02-27T21:44:27.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941398428Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"client":"2","":"downstairs","session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11492024-02-27T21:44:27.557Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941425185Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11502024-02-27T21:44:27.557Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941448002Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11512024-02-27T21:44:27.557Z{"msg":"79d20f19-33ae-49bc-9bf8-88e40928905f is now active with session: 7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941475259Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11522024-02-27T21:44:27.558Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941497197Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178,"session_id":"7acda4f3-2d2c-4d3c-b6ff-a806d470fa8f"}
11532024-02-27T21:44:27.558Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:25.941523793Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178}
11542024-02-27T21:44:27.558ZLoading write count information from file "/tmp/test_repair_verify.data"
11552024-02-27T21:44:27.558ZRun Repair workload
11562024-02-27T21:44:27.558Z01/30 Write block 217 len 32768 data: 4 4 3 3 3 3 2 2
11572024-02-27T21:44:27.558Z02/30 Read block 521 len 12288
11582024-02-27T21:44:27.559Z03/30 Write block 207 len 32768 data: 3 2 2 2 2 2 2 2
11592024-02-27T21:44:27.559Z04/30 Write block 153 len 36864 data: 3 2 2 2 2 2 2 2 2
11602024-02-27T21:44:27.559Z05/30 Read block 37 len 28672
11612024-02-27T21:44:27.570Z06/30 Read block 2 len 28672
11622024-02-27T21:44:27.570Z07/30 Write block 557 len 8192 data: 3 2
11632024-02-27T21:44:27.570Z08/30 Read block 574 len 32768
11642024-02-27T21:44:27.570Z09/30 Flush
11652024-02-27T21:44:27.571Z10/30 Write block 11 len 16384 data: 3 2 2 2
11662024-02-27T21:44:27.571Z11/30 Flush
11672024-02-27T21:44:27.571Z12/30 Write block 312 len 4096 data: 3
11682024-02-27T21:44:27.571Z13/30 Write block 557 len 24576 data: 4 3 2 2 2 2
11692024-02-27T21:44:27.571Z14/30 Write block 306 len 24576 data: 2 2 2 3 3 3
11702024-02-27T21:44:27.571Z15/30 Read block 355 len 12288
11712024-02-27T21:44:27.571Z16/30 Write block 161 len 16384 data: 3 2 2 2
11722024-02-27T21:44:27.571Z17/30 Read block 534 len 40960
11732024-02-27T21:44:27.571Z18/30 Read block 240 len 24576
11742024-02-27T21:44:27.571Z19/30 Write block 328 len 28672 data: 2 2 2 2 2 2 2
11752024-02-27T21:44:27.572Z20/30 Read block 582 len 8192
11762024-02-27T21:44:27.572Z21/30 Read block 284 len 16384
11772024-02-27T21:44:27.572Z22/30 Flush
11782024-02-27T21:44:27.572Z23/30 Write block 568 len 24576 data: 2 3 3 3 3 4
11792024-02-27T21:44:27.572Z24/30 Write block 400 len 8192 data: 2 3
11802024-02-27T21:44:27.572Z25/30 Flush
11812024-02-27T21:44:27.572Z26/30 Write block 258 len 12288 data: 2 2 2
11822024-02-27T21:44:27.572Z27/30 Write block 56 len 12288 data: 2 2 2
11832024-02-27T21:44:27.573Z28/30 Write block 522 len 4096 data: 2
11842024-02-27T21:44:27.573Z29/30 Write block 155 len 12288 data: 3 3 3
11852024-02-27T21:44:27.573Z30/30 Write block 287 len 12288 data: 2 2 3
11862024-02-27T21:44:27.573Z----------------------------------------------------------------
11872024-02-27T21:44:27.573Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11882024-02-27T21:44:27.573ZStates: Active Active Active
11892024-02-27T21:44:27.573ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11902024-02-27T21:44:27.573Z 1 Acked 1000 Write 8 Done Done Done false
11912024-02-27T21:44:27.573Z 2 Acked 1001 Read 3 Done Done Done false
11922024-02-27T21:44:27.574Z 3 Acked 1002 Write 8 Sent Done Done false
11932024-02-27T21:44:27.574Z 4 Acked 1003 Write 9 Sent Done Done false
11942024-02-27T21:44:27.574Z 5 Acked 1004 Read 7 Sent Done Done false
11952024-02-27T21:44:27.574Z 6 Acked 1005 Read 7 Sent Done Done false
11962024-02-27T21:44:27.574Z 7 Acked 1006 Write 2 Sent Done Done false
11972024-02-27T21:44:27.574Z 8 Acked 1007 Read 8 Sent Done Done false
11982024-02-27T21:44:27.575Z 9 Acked 1008 Flush 0 Sent Done Done false
11992024-02-27T21:44:27.575Z 10 Acked 1009 Write 4 Sent Done Done false
12002024-02-27T21:44:27.575Z 11 Acked 1010 Flush 0 Sent Done Done false
12012024-02-27T21:44:27.575Z 12 Acked 1011 Write 1 Sent Done Done false
12022024-02-27T21:44:27.575Z 13 Acked 1012 Write 6 Sent Done Done false
12032024-02-27T21:44:27.576Z 14 Acked 1013 Write 6 Sent Done Done false
12042024-02-27T21:44:27.576Z 15 Acked 1014 Read 3 Sent Done Done false
12052024-02-27T21:44:27.576Z 16 Acked 1015 Write 4 Sent Done Done false
12062024-02-27T21:44:27.576Z 17 Acked 1016 Read 10 Sent Done Done false
12072024-02-27T21:44:27.576Z 18 Acked 1017 Read 6 Sent Done Done false
12082024-02-27T21:44:27.576Z 19 Acked 1018 Write 7 Sent Done Done false
12092024-02-27T21:44:27.577Z 20 Acked 1019 Read 2 Sent Done Done false
12102024-02-27T21:44:27.577Z 21 Acked 1020 Read 4 Sent Done Done false
12112024-02-27T21:44:27.577Z 22 Acked 1021 Flush 0 Sent Done Done false
12122024-02-27T21:44:27.577Z 23 Acked 1022 Write 6 Sent Done Done false
12132024-02-27T21:44:27.577Z 24 Acked 1023 Write 2 Sent Done Done false
12142024-02-27T21:44:27.577Z 25 Acked 1024 Flush 0 Sent Done Done false
12152024-02-27T21:44:27.578Z 26 Acked 1025 Write 3 Sent Done Done false
12162024-02-27T21:44:27.578Z 27 Acked 1026 Write 3 Sent Done Done false
12172024-02-27T21:44:27.578Z 28 Acked 1027 Write 1 Sent Done Done false
12182024-02-27T21:44:27.578Z 29 Acked 1028 Write 3 Sent Done Done false
12192024-02-27T21:44:27.578Z 30 Acked 1029 Write 3 Sent Sent Sent false
12202024-02-27T21:44:27.578Z STATES DS:0 DS:1 DS:2 TOTAL
12212024-02-27T21:44:27.578Z New 0 0 0 0
12222024-02-27T21:44:27.579Z Sent 28 1 1 30
12232024-02-27T21:44:27.579Z Done 2 29 29 60
12242024-02-27T21:44:27.579Z Skipped 0 0 0 0
12252024-02-27T21:44:27.579Z Error 0 0 0 0
12262024-02-27T21:44:27.579ZLast Flush: 0 1024 1024
12272024-02-27T21:44:27.579ZDownstairs last five completed:
12282024-02-27T21:44:27.579ZUpstairs last five completed: 30 29 28 27 26
12292024-02-27T21:44:27.580Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:25.972601743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1178}
12302024-02-27T21:44:27.580ZWrote out file "/tmp/test_repair_verify.data"
12312024-02-27T21:44:27.580Z
12322024-02-27T21:44:27.595ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12332024-02-27T21:44:27.596ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12342024-02-27T21:44:27.596Z 0 000-019 2 10 10 3 18 18 F F F <---
12352024-02-27T21:44:27.596Z 1 020-039 8 8 8 16 16 16 F F F
12362024-02-27T21:44:27.596Z 2 040-059 8 8 8 16 16 16 F T T <---
12372024-02-27T21:44:27.596Z 3 060-079 8 8 8 16 16 16 F F F
12382024-02-27T21:44:27.596Z 4 080-099 6 6 6 14 14 14 F F F
12392024-02-27T21:44:27.596Z 5 100-119 8 8 8 16 16 16 F F F
12402024-02-27T21:44:27.597Z 6 120-139 8 8 8 16 16 16 F F F
12412024-02-27T21:44:27.597Z 7 140-159 4 10 10 8 17 17 F T T <---
12422024-02-27T21:44:27.597Z 8 160-179 8 10 10 16 19 19 F F F <---
12432024-02-27T21:44:27.597Z 9 180-199 8 8 8 16 16 16 F F F
12442024-02-27T21:44:27.597Z 10 200-219 8 10 10 16 17 17 T F F <---
12452024-02-27T21:44:27.597Z 11 220-239 8 10 10 16 17 17 T F F <---
12462024-02-27T21:44:27.597Z 12 240-259 8 8 8 16 16 16 F T T <---
12472024-02-27T21:44:27.598Z 13 260-279 4 4 4 8 8 8 F T T <---
12482024-02-27T21:44:27.598Z 14 280-299 5 5 5 9 9 9 F T T <---
12492024-02-27T21:44:27.598Z 15 300-319 4 10 10 8 19 19 F F F <---
12502024-02-27T21:44:27.598Z 16 320-339 4 10 10 7 19 19 F F F <---
12512024-02-27T21:44:27.598Z 17 340-359 8 8 8 16 16 16 F F F
12522024-02-27T21:44:27.598Z 18 360-379 8 8 8 16 16 16 F F F
12532024-02-27T21:44:27.599Z 19 380-399 8 8 8 16 16 16 F F F
12542024-02-27T21:44:27.599Z 20 400-419 8 10 10 16 20 20 F F F <---
12552024-02-27T21:44:27.599Z 21 420-439 1 1 1 1 1 1 F F F
12562024-02-27T21:44:27.599Z 22 440-459 6 6 6 14 14 14 F F F
12572024-02-27T21:44:27.599Z 23 460-479 8 8 8 16 16 16 F F F
12582024-02-27T21:44:27.599Z 24 480-499 6 6 6 12 12 12 F F F
12592024-02-27T21:44:27.599Z 25 500-519 2 2 2 4 4 4 F F F
12602024-02-27T21:44:27.599Z 26 520-539 1 1 1 1 1 1 F T T <---
12612024-02-27T21:44:27.600Z 27 540-559 8 10 10 16 19 19 F F F <---
12622024-02-27T21:44:27.600Z 28 560-579 7 10 10 15 20 20 F F F <---
12632024-02-27T21:44:27.600Z 29 580-599 7 7 7 15 15 15 F F F
12642024-02-27T21:44:27.600ZMax gen: 10, Max flush: 20
12652024-02-27T21:44:27.600ZError: Difference in extent metadata found!
12662024-02-27T21:44:27.601ZA Difference in extent metadata is expected here
12672024-02-27T21:44:27.601ZOn loop 5
12682024-02-27T21:44:27.601Z
12692024-02-27T21:44:27.615ZVerifying data now
12702024-02-27T21:44:32.327ZNo change after verify
12712024-02-27T21:44:32.327ZLoop: 5 Downstairs dump after verify (and repair):
12722024-02-27T21:44:32.368ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12732024-02-27T21:44:32.368Z 0 000-019 10 10 10 18 18 18 F F F
12742024-02-27T21:44:32.368Z 1 020-039 8 8 8 16 16 16 F F F
12752024-02-27T21:44:32.368Z 2 040-059 11 11 11 21 21 21 F F F
12762024-02-27T21:44:32.368Z 3 060-079 8 8 8 16 16 16 F F F
12772024-02-27T21:44:32.368Z 4 080-099 6 6 6 14 14 14 F F F
12782024-02-27T21:44:32.368Z 5 100-119 8 8 8 16 16 16 F F F
12792024-02-27T21:44:32.368Z 6 120-139 8 8 8 16 16 16 F F F
12802024-02-27T21:44:32.368Z 7 140-159 11 11 11 21 21 21 F F F
12812024-02-27T21:44:32.369Z 8 160-179 10 10 10 19 19 19 F F F
12822024-02-27T21:44:32.369Z 9 180-199 8 8 8 16 16 16 F F F
12832024-02-27T21:44:32.369Z 10 200-219 10 10 10 17 17 17 F F F
12842024-02-27T21:44:32.369Z 11 220-239 10 10 10 17 17 17 F F F
12852024-02-27T21:44:32.369Z 12 240-259 11 11 11 21 21 21 F F F
12862024-02-27T21:44:32.369Z 13 260-279 11 11 11 21 21 21 F F F
12872024-02-27T21:44:32.369Z 14 280-299 11 11 11 21 21 21 F F F
12882024-02-27T21:44:32.369Z 15 300-319 10 10 10 19 19 19 F F F
12892024-02-27T21:44:32.369Z 16 320-339 10 10 10 19 19 19 F F F
12902024-02-27T21:44:32.369Z 17 340-359 8 8 8 16 16 16 F F F
12912024-02-27T21:44:32.369Z 18 360-379 8 8 8 16 16 16 F F F
12922024-02-27T21:44:32.370Z 19 380-399 8 8 8 16 16 16 F F F
12932024-02-27T21:44:32.370Z 20 400-419 10 10 10 20 20 20 F F F
12942024-02-27T21:44:32.370Z 21 420-439 1 1 1 1 1 1 F F F
12952024-02-27T21:44:32.370Z 22 440-459 6 6 6 14 14 14 F F F
12962024-02-27T21:44:32.370Z 23 460-479 8 8 8 16 16 16 F F F
12972024-02-27T21:44:32.370Z 24 480-499 6 6 6 12 12 12 F F F
12982024-02-27T21:44:32.370Z 25 500-519 2 2 2 4 4 4 F F F
12992024-02-27T21:44:32.370Z 26 520-539 11 11 11 21 21 21 F F F
13002024-02-27T21:44:32.370Z 27 540-559 10 10 10 19 19 19 F F F
13012024-02-27T21:44:32.370Z 28 560-579 10 10 10 20 20 20 F F F
13022024-02-27T21:44:32.370Z 29 580-599 7 7 7 15 15 15 F F F
13032024-02-27T21:44:32.371ZMax gen: 11, Max flush: 21
13042024-02-27T21:44:32.371Z
13052024-02-27T21:44:32.371ZBegin loop 6
13062024-02-27T21:44:32.371ZDownstairs to restart: 0
13072024-02-27T21:44:32.371Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753138122Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186}
13082024-02-27T21:44:32.371Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753437976Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186}
13092024-02-27T21:44:32.371Z{"msg":"Crucible runtime is spawned
13102024-02-27T21:44:32.371ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753529856Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186}
13112024-02-27T21:44:32.371Z{"msg":"Crucible a9f6878d-0a21-45f9-b9c1-c9d734662f46 has session id: 10bd63af-774a-4d17-8427-277528be1161","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753570941Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13122024-02-27T21:44:32.371Z{"msg":"Upstairs opts: Upstairs UUID: a9f6878d-0a21-45f9-b9c1-c9d734662f46, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753625314Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13132024-02-27T21:44:32.371Z{"msg":"Crucible stats registered with UUID: a9f6878d-0a21-45f9-b9c1-c9d734662f46","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753655081Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13142024-02-27T21:44:32.371Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753687857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186}
13152024-02-27T21:44:32.371Z{"msg":"a9f6878d-0a21-45f9-b9c1-c9d734662f46 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753703375Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13162024-02-27T21:44:32.372Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753719583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13172024-02-27T21:44:32.372Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.75374496Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13182024-02-27T21:44:32.372Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753763318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13192024-02-27T21:44:32.372Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753786585Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13202024-02-27T21:44:32.372Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753806743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13212024-02-27T21:44:32.372Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753824341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13222024-02-27T21:44:32.372Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753842709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13232024-02-27T21:44:32.373Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.753986962Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13242024-02-27T21:44:32.373Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.754034496Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13252024-02-27T21:44:32.373Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.754061073Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"io task","client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13262024-02-27T21:44:32.373Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.75408602Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"io task","client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13272024-02-27T21:44:32.373Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.754111277Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"io task","client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13282024-02-27T21:44:32.373Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.754774839Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13292024-02-27T21:44:32.373Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.754843511Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13302024-02-27T21:44:32.373Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.754885196Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13312024-02-27T21:44:32.373Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.754904843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13322024-02-27T21:44:32.373Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.754949038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13332024-02-27T21:44:32.373Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.754993123Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13342024-02-27T21:44:32.374Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.75544355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13352024-02-27T21:44:32.374Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.755481166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13362024-02-27T21:44:32.374Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.755557287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13372024-02-27T21:44:32.374Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756013833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13382024-02-27T21:44:32.374Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756058398Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13392024-02-27T21:44:32.374Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756077576Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13402024-02-27T21:44:32.374Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756164455Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13412024-02-27T21:44:32.374Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756192142Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13422024-02-27T21:44:32.374Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.75620891Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13432024-02-27T21:44:32.374Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756237966Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13442024-02-27T21:44:32.374Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756259774Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13452024-02-27T21:44:32.374Z{"msg":"[0]R flush_numbers[0..12]: [18, 16, 21, 16, 14, 16, 16, 21, 19, 16, 17, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.75628884Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13462024-02-27T21:44:32.374Z{"msg":"[0]R generation[0..12]: [10, 8, 11, 8, 6, 8, 8, 11, 10, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756326486Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13472024-02-27T21:44:32.375ZWait for a query_work_queue command to finish before sending IO
13482024-02-27T21:44:32.375Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756358152Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13492024-02-27T21:44:32.375Z{"msg":"[1]R flush_numbers[0..12]: [18, 16, 21, 16, 14, 16, 16, 21, 19, 16, 17, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756392728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13502024-02-27T21:44:32.375Z{"msg":"[1]R generation[0..12]: [10, 8, 11, 8, 6, 8, 8, 11, 10, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756427434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13512024-02-27T21:44:32.375Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.75646147Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13522024-02-27T21:44:32.375Z{"msg":"[2]R flush_numbers[0..12]: [18, 16, 21, 16, 14, 16, 16, 21, 19, 16, 17, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756496396Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13532024-02-27T21:44:32.375Z{"msg":"[2]R generation[0..12]: [10, 8, 11, 8, 6, 8, 8, 11, 10, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756528182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13542024-02-27T21:44:32.375Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756553579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13552024-02-27T21:44:32.375Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756593525Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13562024-02-27T21:44:32.375Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756619042Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13572024-02-27T21:44:32.375Z{"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756647958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13582024-02-27T21:44:32.375Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756671766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"":"downstairs","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13592024-02-27T21:44:32.375Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756698942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13602024-02-27T21:44:32.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.75672038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13612024-02-27T21:44:32.376Z,"pid":1186,"client":"0","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13622024-02-27T21:44:32.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756760605Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"1","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13632024-02-27T21:44:32.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756785172Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"client":"2","":"downstairs","session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13642024-02-27T21:44:32.376Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756817678Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13652024-02-27T21:44:32.376Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756840715Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13662024-02-27T21:44:32.376Z{"msg":"a9f6878d-0a21-45f9-b9c1-c9d734662f46 is now active with session: 10bd63af-774a-4d17-8427-277528be1161","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756867532Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13672024-02-27T21:44:32.376Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.75688901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186,"session_id":"10bd63af-774a-4d17-8427-277528be1161"}
13682024-02-27T21:44:32.376Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:30.756914487Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186}
13692024-02-27T21:44:32.376ZLoading write count information from file "/tmp/test_repair_verify.data"
13702024-02-27T21:44:32.376ZRun Repair workload
13712024-02-27T21:44:32.431Z01/30 Write block 286 len 36864 data: 2 3 3 4 3 2 2 2 2
13722024-02-27T21:44:32.431Z02/30 Flush
13732024-02-27T21:44:32.431Z03/30 Flush
13742024-02-27T21:44:32.431Z04/30 Write block 71 len 20480 data: 2 2 2 2 2
13752024-02-27T21:44:32.431Z05/30 Flush
13762024-02-27T21:44:32.431Z06/30 Flush
13772024-02-27T21:44:32.432Z07/30 Write block 459 len 4096 data: 2
13782024-02-27T21:44:32.432Z08/30 Read block 561 len 24576
13792024-02-27T21:44:32.432Z09/30 Flush
13802024-02-27T21:44:32.432Z10/30 Write block 172 len 28672 data: 4 4 4 3 2 2 2
13812024-02-27T21:44:32.432Z11/30 Write block 56 len 32768 data: 3 3 3 2 2 2 2 2
13822024-02-27T21:44:32.433Z12/30 Read block 54 len 36864
13832024-02-27T21:44:32.433Z13/30 Write block 72 len 32768 data: 3 3 3 3 2 3 3 3
13842024-02-27T21:44:32.433Z14/30 Read block 290 len 40960
13852024-02-27T21:44:32.433Z15/30 Flush
13862024-02-27T21:44:32.433Z16/30 Write block 332 len 24576 data: 3 3 3 2 2 3
13872024-02-27T21:44:32.433Z17/30 Read block 87 len 16384
13882024-02-27T21:44:32.433Z18/30 Write block 247 len 40960 data: 3 2 2 2 2 2 2 2 2 2
13892024-02-27T21:44:32.433Z19/30 Read block 300 len 20480
13902024-02-27T21:44:32.433Z20/30 Write block 75 len 24576 data: 4 3 4 4 4 2
13912024-02-27T21:44:32.433Z21/30 Read block 254 len 20480
13922024-02-27T21:44:32.434Z22/30 Write block 341 len 24576 data: 3 3 3 3 2 2
13932024-02-27T21:44:32.434Z23/30 Read block 77 len 4096
13942024-02-27T21:44:32.434Z24/30 Read block 297 len 20480
13952024-02-27T21:44:32.434Z25/30 Read block 218 len 32768
13962024-02-27T21:44:32.434Z26/30 Read block 9 len 32768
13972024-02-27T21:44:32.434Z27/30 Write block 307 len 40960 data: 3 3 4 4 4 4 3 3 3 3
13982024-02-27T21:44:32.434Z28/30 Read block 549 len 40960
13992024-02-27T21:44:32.434Z29/30 Write block 581 len 28672 data: 2 2 2 2 2 2 2
14002024-02-27T21:44:32.435Z30/30 Read block 90 len 24576
14012024-02-27T21:44:32.435Z----------------------------------------------------------------
14022024-02-27T21:44:32.435Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:27
14032024-02-27T21:44:32.435ZStates: Active Active Active
14042024-02-27T21:44:32.435ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14052024-02-27T21:44:32.435Z 4 Acked 1003 Write 5 Sent Done Done false
14062024-02-27T21:44:32.435Z 5 Acked 1004 Flush 0 Sent Done Done false
14072024-02-27T21:44:32.435Z 6 Acked 1005 Flush 0 Sent Done Done false
14082024-02-27T21:44:32.435Z 7 Acked 1006 Write 1 Sent Done Done false
14092024-02-27T21:44:32.435Z 8 Acked 1007 Read 6 Sent Done Done false
14102024-02-27T21:44:32.435Z 9 Acked 1008 Flush 0 Sent Done Done false
14112024-02-27T21:44:32.435Z 10 Acked 1009 Write 7 Sent Done Done false
14122024-02-27T21:44:32.436Z 11 Acked 1010 Write 8 Sent Done Done false
14132024-02-27T21:44:32.436Z 12 Acked 1011 Read 9 Sent Done Done false
14142024-02-27T21:44:32.436Z 13 Acked 1012 Write 8 Sent Done Done false
14152024-02-27T21:44:32.436Z 14 Acked 1013 Read 10 Sent Done Done false
14162024-02-27T21:44:32.436Z 15 Acked 1014 Flush 0 Sent Done Done false
14172024-02-27T21:44:32.436Z 16 Acked 1015 Write 6 Sent Done Done false
14182024-02-27T21:44:32.436Z 17 Acked 1016 Read 4 Sent Done Done false
14192024-02-27T21:44:32.436Z 18 Acked 1017 Write 10 Sent Done Done false
14202024-02-27T21:44:32.436Z 19 Acked 1018 Read 5 Sent Done Done false
14212024-02-27T21:44:32.436Z 20 Acked 1019 Write 6 Sent Done Done false
14222024-02-27T21:44:32.436Z 21 Acked 1020 Read 5 Sent Done Done false
14232024-02-27T21:44:32.437Z 22 Acked 1021 Write 6 Sent Done Done false
14242024-02-27T21:44:32.437Z 23 Acked 1022 Read 1 Sent Done Done false
14252024-02-27T21:44:32.437Z 24 Acked 1023 Read 5 Sent Done Done false
14262024-02-27T21:44:32.437Z 25 Acked 1024 Read 8 Sent Done Done false
14272024-02-27T21:44:32.437Z 26 Acked 1025 Read 8 Sent Done Done false
14282024-02-27T21:44:32.437Z 27 Acked 1026 Write 10 Sent Done Done false
14292024-02-27T21:44:32.437Z 28 Acked 1027 Read 10 Sent Done Done false
14302024-02-27T21:44:32.437Z 29 Acked 1028 Write 7 Sent Done Done false
14312024-02-27T21:44:32.437Z 30 Acked 1029 Read 6 Sent Sent Done false
14322024-02-27T21:44:32.437Z STATES DS:0 DS:1 DS:2 TOTAL
14332024-02-27T21:44:32.437Z New 0 0 0 0
14342024-02-27T21:44:32.438Z Sent 27 1 0 28
14352024-02-27T21:44:32.438Z Done 0 26 27 53
14362024-02-27T21:44:32.438Z Skipped 0 0 0 0
14372024-02-27T21:44:32.438Z Error 0 0 0 0
14382024-02-27T21:44:32.438ZLast Flush: 1002 1014 1014
14392024-02-27T21:44:32.438ZDownstairs last five completed: 1002 1001 1000
14402024-02-27T21:44:32.438ZUpstairs last five completed: 30 29 28 27 26
14412024-02-27T21:44:32.439Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:30.792235606Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1186}
14422024-02-27T21:44:32.439ZWrote out file "/tmp/test_repair_verify.data"
14432024-02-27T21:44:32.439Z
14442024-02-27T21:44:32.439ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14452024-02-27T21:44:32.439ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14462024-02-27T21:44:32.439Z 0 000-019 10 10 10 18 18 18 F F F
14472024-02-27T21:44:32.439Z 1 020-039 8 8 8 16 16 16 F F F
14482024-02-27T21:44:32.439Z 2 040-059 11 12 12 21 27 27 F F F <---
14492024-02-27T21:44:32.439Z 3 060-079 8 12 12 16 27 27 F T T <---
14502024-02-27T21:44:32.440Z 4 080-099 6 6 6 14 14 14 F T T <---
14512024-02-27T21:44:32.440Z 5 100-119 8 8 8 16 16 16 F F F
14522024-02-27T21:44:32.440Z 6 120-139 8 8 8 16 16 16 F F F
14532024-02-27T21:44:32.440Z 7 140-159 11 11 11 21 21 21 F F F
14542024-02-27T21:44:32.440Z 8 160-179 10 12 12 19 27 27 F F F <---
14552024-02-27T21:44:32.440Z 9 180-199 8 8 8 16 16 16 F F F
14562024-02-27T21:44:32.440Z 10 200-219 10 10 10 17 17 17 F F F
14572024-02-27T21:44:32.440Z 11 220-239 10 10 10 17 17 17 F F F
14582024-02-27T21:44:32.440Z 12 240-259 11 11 11 21 21 21 F T T <---
14592024-02-27T21:44:32.441Z 13 260-279 11 11 11 21 21 21 F F F
14602024-02-27T21:44:32.441Z 14 280-299 12 12 12 22 22 22 F F F
14612024-02-27T21:44:32.441Z 15 300-319 10 10 10 19 19 19 F T T <---
14622024-02-27T21:44:32.441Z 16 320-339 10 10 10 19 19 19 F T T <---
14632024-02-27T21:44:32.441Z 17 340-359 8 8 8 16 16 16 F T T <---
14642024-02-27T21:44:32.441Z 18 360-379 8 8 8 16 16 16 F F F
14652024-02-27T21:44:32.442Z 19 380-399 8 8 8 16 16 16 F F F
14662024-02-27T21:44:32.442Z 20 400-419 10 10 10 20 20 20 F F F
14672024-02-27T21:44:32.442Z 21 420-439 1 1 1 1 1 1 F F F
14682024-02-27T21:44:32.442Z 22 440-459 6 12 12 14 26 26 F F F <---
14692024-02-27T21:44:32.442Z 23 460-479 8 8 8 16 16 16 F F F
14702024-02-27T21:44:32.442Z 24 480-499 6 6 6 12 12 12 F F F
14712024-02-27T21:44:32.464Z 25 500-519 2 2 2 4 4 4 F F F
14722024-02-27T21:44:32.464Z 26 520-539 11 11 11 21 21 21 F F F
14732024-02-27T21:44:32.464Z 27 540-559 10 10 10 19 19 19 F F F
14742024-02-27T21:44:32.464Z 28 560-579 10 10 10 20 20 20 F F F
14752024-02-27T21:44:32.465Z 29 580-599 7 7 7 15 15 15 F T T <---
14762024-02-27T21:44:32.465ZMax gen: 12, Max flush: 27
14772024-02-27T21:44:32.465ZError: Difference in extent metadata found!
14782024-02-27T21:44:32.465ZA Difference in extent metadata is expected here
14792024-02-27T21:44:32.466ZOn loop 6
14802024-02-27T21:44:32.466Z
14812024-02-27T21:44:32.466ZVerifying data now
14822024-02-27T21:44:36.940ZNo change after verify
14832024-02-27T21:44:36.940ZLoop: 6 Downstairs dump after verify (and repair):
14842024-02-27T21:44:36.957ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14852024-02-27T21:44:36.957Z 0 000-019 10 10 10 18 18 18 F F F
14862024-02-27T21:44:36.957Z 1 020-039 8 8 8 16 16 16 F F F
14872024-02-27T21:44:36.958Z 2 040-059 12 12 12 27 27 27 F F F
14882024-02-27T21:44:36.958Z 3 060-079 13 13 13 28 28 28 F F F
14892024-02-27T21:44:36.958Z 4 080-099 13 13 13 28 28 28 F F F
14902024-02-27T21:44:36.958Z 5 100-119 8 8 8 16 16 16 F F F
14912024-02-27T21:44:36.958Z 6 120-139 8 8 8 16 16 16 F F F
14922024-02-27T21:44:36.958Z 7 140-159 11 11 11 21 21 21 F F F
14932024-02-27T21:44:36.958Z 8 160-179 12 12 12 27 27 27 F F F
14942024-02-27T21:44:36.958Z 9 180-199 8 8 8 16 16 16 F F F
14952024-02-27T21:44:36.958Z 10 200-219 10 10 10 17 17 17 F F F
14962024-02-27T21:44:36.958Z 11 220-239 10 10 10 17 17 17 F F F
14972024-02-27T21:44:36.958Z 12 240-259 13 13 13 28 28 28 F F F
14982024-02-27T21:44:36.959Z 13 260-279 11 11 11 21 21 21 F F F
14992024-02-27T21:44:36.959Z 14 280-299 12 12 12 22 22 22 F F F
15002024-02-27T21:44:36.959Z 15 300-319 13 13 13 28 28 28 F F F
15012024-02-27T21:44:36.959Z 16 320-339 13 13 13 28 28 28 F F F
15022024-02-27T21:44:36.959Z 17 340-359 13 13 13 28 28 28 F F F
15032024-02-27T21:44:36.959Z 18 360-379 8 8 8 16 16 16 F F F
15042024-02-27T21:44:36.959Z 19 380-399 8 8 8 16 16 16 F F F
15052024-02-27T21:44:36.959Z 20 400-419 10 10 10 20 20 20 F F F
15062024-02-27T21:44:36.959Z 21 420-439 1 1 1 1 1 1 F F F
15072024-02-27T21:44:36.959Z 22 440-459 12 12 12 26 26 26 F F F
15082024-02-27T21:44:36.959Z 23 460-479 8 8 8 16 16 16 F F F
15092024-02-27T21:44:36.959Z 24 480-499 6 6 6 12 12 12 F F F
15102024-02-27T21:44:36.959Z 25 500-519 2 2 2 4 4 4 F F F
15112024-02-27T21:44:36.959Z 26 520-539 11 11 11 21 21 21 F F F
15122024-02-27T21:44:36.959Z 27 540-559 10 10 10 19 19 19 F F F
15132024-02-27T21:44:36.960Z 28 560-579 10 10 10 20 20 20 F F F
15142024-02-27T21:44:36.960Z 29 580-599 13 13 13 28 28 28 F F F
15152024-02-27T21:44:36.960ZMax gen: 13, Max flush: 28
15162024-02-27T21:44:36.960Z
15172024-02-27T21:44:36.960ZBegin loop 7
15182024-02-27T21:44:36.960ZDownstairs to restart: 0
15192024-02-27T21:44:37.429Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.378988373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194}
15202024-02-27T21:44:37.429Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":Crucible runtime is spawned
15212024-02-27T21:44:37.429Z0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379313204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194}
15222024-02-27T21:44:37.429Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.37934821Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194}
15232024-02-27T21:44:37.429Z{"msg":"Crucible bb0af210-5ff5-4368-a5ee-d05a38ebdc62 has session id: 8211ede9-9323-47ca-8116-85db648033e8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379387775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15242024-02-27T21:44:37.429Z{"msg":"Upstairs opts: Upstairs UUID: bb0af210-5ff5-4368-a5ee-d05a38ebdc62, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.37943333Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15252024-02-27T21:44:37.429Z{"msg":"Crucible stats registered with UUID: bb0af210-5ff5-4368-a5ee-d05a38ebdc62","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379465426Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15262024-02-27T21:44:37.430Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379511901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194}
15272024-02-27T21:44:37.430Z{"msg":"bb0af210-5ff5-4368-a5ee-d05a38ebdc62 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379546717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15282024-02-27T21:44:37.430Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379565454Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15292024-02-27T21:44:37.430Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379615439Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15302024-02-27T21:44:37.430Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379660783Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15312024-02-27T21:44:37.430Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.37969056Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15322024-02-27T21:44:37.430Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379732675Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15332024-02-27T21:44:37.430Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379754462Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15342024-02-27T21:44:37.430Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.3797717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15352024-02-27T21:44:37.430Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379813765Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15362024-02-27T21:44:37.430Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.379843432Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15372024-02-27T21:44:37.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.380025271Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15382024-02-27T21:44:37.431Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.380071265Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15392024-02-27T21:44:37.431Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.380100122Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15402024-02-27T21:44:37.431Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.380752885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15412024-02-27T21:44:37.431Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.380800629Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15422024-02-27T21:44:37.431Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.380851203Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15432024-02-27T21:44:37.431Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.380898857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15442024-02-27T21:44:37.431Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.381483949Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15452024-02-27T21:44:37.431Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.381533743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15462024-02-27T21:44:37.431Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.381606304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15472024-02-27T21:44:37.431Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.381652439Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15482024-02-27T21:44:37.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382041343Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15492024-02-27T21:44:37.432Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382081598Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15502024-02-27T21:44:37.432Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382125303Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15512024-02-27T21:44:37.432Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382204444Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15522024-02-27T21:44:37.432Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382250238Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15532024-02-27T21:44:37.432Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382288914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15542024-02-27T21:44:37.432Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382428237Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15552024-02-27T21:44:37.432Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382854757Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15562024-02-27T21:44:37.432Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382904371Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15572024-02-27T21:44:37.432Z{"msg":"[0]R flush_numbers[0..12]: [18, 16, 27, 28, 28, 16, 16, 21, 27, 16, 17, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382926119Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15582024-02-27T21:44:37.432Z{"msg":"[0]R generation[0..12]: [10, 8, 12, 13, 13, 8, 8, 11, 12, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382950676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15592024-02-27T21:44:37.433Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382966154Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15602024-02-27T21:44:37.433Z{"msg":"[1]R flush_numbers[0..12]: [18, 16, 27, 28, 28, 16, 16, 21, 27, 16, 17, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.382980672Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15612024-02-27T21:44:37.433Z{"msg":"[1]R generation[0..12]: [10, 8, 12, 13, 13, 8, 8, 11, 12, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383006029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
15622024-02-27T21:44:37.433Zsession_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15632024-02-27T21:44:37.433Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383042585Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15642024-02-27T21:44:37.433Z{"msg":"[2]R flush_numbers[0..12]: [18, 16, 27, 28, 28, 16, 16, 21, 27, 16, 17, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383057283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15652024-02-27T21:44:37.433Z{"msg":"[2]R generation[0..12]: [10, 8, 12, 13, 13, 8, 8, 11, 12, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383078791Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15662024-02-27T21:44:37.433Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383096479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15672024-02-27T21:44:37.434Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383130065Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15682024-02-27T21:44:37.434Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383157382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15692024-02-27T21:44:37.434Z{"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383182728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15702024-02-27T21:44:37.434Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383215635Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"":"downstairs","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15712024-02-27T21:44:37.434Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383241612Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15722024-02-27T21:44:37.434Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383261599Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"0","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15732024-02-27T21:44:37.434Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383292326Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"1","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15742024-02-27T21:44:37.434Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383327562Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"client":"2","":"downstairs","session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15752024-02-27T21:44:37.434Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383362977Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15762024-02-27T21:44:37.435Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383379725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15772024-02-27T21:44:37.435Z{"msg":"bb0af210-5ff5-4368-a5ee-d05a38ebdc62 is now active with session: 8211ede9-9323-47ca-8116-85db648033e8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383411381Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15782024-02-27T21:44:37.435Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383436678Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194,"session_id":"8211ede9-9323-47ca-8116-85db648033e8"}
15792024-02-27T21:44:37.435Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:35.383460926Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194}
15802024-02-27T21:44:37.446ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15812024-02-27T21:44:37.446ZLoading write count information from file "/tmp/test_repair_verify.data"
15822024-02-27T21:44:37.446ZRun Repair workload
15832024-02-27T21:44:37.446Z01/30 Write block 551 len 20480 data: 3 3 3 3 3
15842024-02-27T21:44:37.446Z02/30 Read block 189 len 12288
15852024-02-27T21:44:37.447Z03/30 Flush
15862024-02-27T21:44:37.447Z04/30 Write block 516 len 8192 data: 2 2
15872024-02-27T21:44:37.447Z05/30 Read block 361 len 36864
15882024-02-27T21:44:37.447Z06/30 Write block 331 len 20480 data: 3 4 4 4 3
15892024-02-27T21:44:37.447Z07/30 Write block 208 len 20480 data: 3 3 3 3 3
15902024-02-27T21:44:37.447Z08/30 Write block 561 len 12288 data: 3 3 2
15912024-02-27T21:44:37.447Z09/30 Read block 352 len 28672
15922024-02-27T21:44:37.447Z10/30 Flush
15932024-02-27T21:44:37.448Z11/30 Write block 429 len 16384 data: 2 2 2 2
15942024-02-27T21:44:37.448Z12/30 Write block 230 len 28672 data: 4 4 4 4 3 3 2
15952024-02-27T21:44:37.448Z13/30 Read block 77 len 40960
15962024-02-27T21:44:37.448Z14/30 Read block 341 len 4096
15972024-02-27T21:44:37.448Z15/30 Read block 226 len 28672
15982024-02-27T21:44:37.448Z16/30 Flush
15992024-02-27T21:44:37.448Z17/30 Write block 510 len 32768 data: 2 2 2 2 2 2 3 3
16002024-02-27T21:44:37.448Z18/30 Read block 449 len 32768
16012024-02-27T21:44:37.448Z19/30 Read block 138 len 20480
16022024-02-27T21:44:37.448Z20/30 Read block 61 len 20480
16032024-02-27T21:44:37.448Z21/30 Read block 61 len 20480
16042024-02-27T21:44:37.449Z22/30 Read block 104 len 16384
16052024-02-27T21:44:37.449Z23/30 Read block 401 len 20480
16062024-02-27T21:44:37.449Z24/30 Flush
16072024-02-27T21:44:37.449Z25/30 Write block 81 len 36864 data: 2 2 2 2 2 2 3 4 4
16082024-02-27T21:44:37.449Z26/30 Read block 91 len 20480
16092024-02-27T21:44:37.449Z27/30 Read block 40 len 28672
16102024-02-27T21:44:37.449Z28/30 Read block 363 len 28672
16112024-02-27T21:44:37.449Z29/30 Write block 433 len 20480 data: 2 2 2 2 2
16122024-02-27T21:44:37.449Z30/30 Read block 433 len 36864
16132024-02-27T21:44:37.449Z----------------------------------------------------------------
16142024-02-27T21:44:37.450Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16152024-02-27T21:44:37.450ZStates: Active Active Active
16162024-02-27T21:44:37.450ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16172024-02-27T21:44:37.450Z 1 Acked 1000 Write 5 Done Done Done false
16182024-02-27T21:44:37.450Z 2 Acked 1001 Read 3 Done Done Done false
16192024-02-27T21:44:37.450Z 3 Acked 1002 Flush 0 Sent Done Done false
16202024-02-27T21:44:37.450Z 4 Acked 1003 Write 2 Sent Done Done false
16212024-02-27T21:44:37.450Z 5 Acked 1004 Read 9 Sent Done Done false
16222024-02-27T21:44:37.450Z 6 Acked 1005 Write 5 Sent Done Done false
16232024-02-27T21:44:37.450Z 7 Acked 1006 Write 5 Sent Done Done false
16242024-02-27T21:44:37.450Z 8 Acked 1007 Write 3 Sent Done Done false
16252024-02-27T21:44:37.451Z 9 Acked 1008 Read 7 Sent Done Done false
16262024-02-27T21:44:37.451Z 10 Acked 1009 Flush 0 Sent Done Done false
16272024-02-27T21:44:37.451Z 11 Acked 1010 Write 4 Sent Done Done false
16282024-02-27T21:44:37.451Z 12 Acked 1011 Write 7 Sent Done Done false
16292024-02-27T21:44:37.451Z 13 Acked 1012 Read 10 Sent Done Done false
16302024-02-27T21:44:37.451Z 14 Acked 1013 Read 1 Sent Done Done false
16312024-02-27T21:44:37.451Z 15 Acked 1014 Read 7 Sent Done Done false
16322024-02-27T21:44:37.451Z 16 Acked 1015 Flush 0 Sent Done Done false
16332024-02-27T21:44:37.451Z 17 Acked 1016 Write 8 Sent Done Done false
16342024-02-27T21:44:37.451Z 18 Acked 1017 Read 8 Sent Done Done false
16352024-02-27T21:44:37.451Z 19 Acked 1018 Read 5 Sent Done Done false
16362024-02-27T21:44:37.451Z 20 Acked 1019 Read 5 Sent Done Done false
16372024-02-27T21:44:37.452Z 21 Acked 1020 Read 5 Sent Done Done false
16382024-02-27T21:44:37.452Z 22 Acked 1021 Read 4 Sent Done Done false
16392024-02-27T21:44:37.452Z 23 Acked 1022 Read 5 Sent Done Done false
16402024-02-27T21:44:37.452Z 24 Acked 1023 Flush 0 Sent Done Done false
16412024-02-27T21:44:37.452Z 25 Acked 1024 Write 9 Sent Done Done false
16422024-02-27T21:44:37.452Z 26 Acked 1025 Read 5 Sent Done Done false
16432024-02-27T21:44:37.452Z 27 Acked 1026 Read 7 Sent Done Done false
16442024-02-27T21:44:37.452Z 28 Acked 1027 Read 7 Sent Done Done false
16452024-02-27T21:44:37.452Z 29 Acked 1028 Write 5 Sent Done Done false
16462024-02-27T21:44:37.453Z 30 Acked 1029 Read 9 Sent Sent Done false
16472024-02-27T21:44:37.453Z STATES DS:0 DS:1 DS:2 TOTAL
16482024-02-27T21:44:37.453Z New 0 0 0 0
16492024-02-27T21:44:37.453Z Sent 28 1 0 29
16502024-02-27T21:44:37.453Z Done 2 29 30 61
16512024-02-27T21:44:37.453Z Skipped 0 0 0 0
16522024-02-27T21:44:37.453Z Error 0 0 0 0
16532024-02-27T21:44:37.453ZLast Flush: 0 1023 1023
16542024-02-27T21:44:37.453ZDownstairs last five completed:
16552024-02-27T21:44:37.453ZUpstairs last five completed: 30 29 28 27 26
16562024-02-27T21:44:37.453Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:35.415816764Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1194}
16572024-02-27T21:44:37.453ZWrote out file "/tmp/test_repair_verify.data"
16582024-02-27T21:44:37.453Z
16592024-02-27T21:44:37.454ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16602024-02-27T21:44:37.454ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16612024-02-27T21:44:37.454Z 0 000-019 10 10 10 18 18 18 F F F
16622024-02-27T21:44:37.454Z 1 020-039 8 8 8 16 16 16 F F F
16632024-02-27T21:44:37.454Z 2 040-059 12 12 12 27 27 27 F F F
16642024-02-27T21:44:37.454Z 3 060-079 13 13 13 28 28 28 F F F
16652024-02-27T21:44:37.454Z 4 080-099 13 13 13 28 28 28 F T T <---
16662024-02-27T21:44:37.454Z 5 100-119 8 8 8 16 16 16 F F F
16672024-02-27T21:44:37.454Z 6 120-139 8 8 8 16 16 16 F F F
16682024-02-27T21:44:37.454Z 7 140-159 11 11 11 21 21 21 F F F
16692024-02-27T21:44:37.455Z 8 160-179 12 12 12 27 27 27 F F F
16702024-02-27T21:44:37.455Z 9 180-199 8 8 8 16 16 16 F F F
16712024-02-27T21:44:37.455Z 10 200-219 10 14 14 17 30 30 F F F <---
16722024-02-27T21:44:37.455Z 11 220-239 10 14 14 17 31 31 F F F <---
16732024-02-27T21:44:37.455Z 12 240-259 13 13 13 28 28 28 F F F
16742024-02-27T21:44:37.455Z 13 260-279 11 11 11 21 21 21 F F F
16752024-02-27T21:44:37.455Z 14 280-299 12 12 12 22 22 22 F F F
16762024-02-27T21:44:37.455Z 15 300-319 13 13 13 28 28 28 F F F
16772024-02-27T21:44:37.455Z 16 320-339 13 14 14 28 30 30 F F F <---
16782024-02-27T21:44:37.455Z 17 340-359 13 13 13 28 28 28 F F F
16792024-02-27T21:44:37.455Z 18 360-379 8 8 8 16 16 16 F F F
16802024-02-27T21:44:37.465Z 19 380-399 8 8 8 16 16 16 F F F
16812024-02-27T21:44:37.465Z 20 400-419 10 10 10 20 20 20 F F F
16822024-02-27T21:44:37.466Z 21 420-439 1 14 14 1 31 31 F T T <---
16832024-02-27T21:44:37.466Z 22 440-459 12 12 12 26 26 26 F F F
16842024-02-27T21:44:37.466Z 23 460-479 8 8 8 16 16 16 F F F
16852024-02-27T21:44:37.467Z 24 480-499 6 6 6 12 12 12 F F F
16862024-02-27T21:44:37.467Z 25 500-519 2 14 14 4 32 32 F F F <---
16872024-02-27T21:44:37.468Z 26 520-539 11 11 11 21 21 21 F F F
16882024-02-27T21:44:37.468Z 27 540-559 10 14 14 19 29 29 T F F <---
16892024-02-27T21:44:37.468Z 28 560-579 10 14 14 20 30 30 F F F <---
16902024-02-27T21:44:37.468Z 29 580-599 13 13 13 28 28 28 F F F
16912024-02-27T21:44:37.468ZMax gen: 14, Max flush: 32
16922024-02-27T21:44:37.469ZError: Difference in extent metadata found!
16932024-02-27T21:44:37.469ZA Difference in extent metadata is expected here
16942024-02-27T21:44:37.469ZOn loop 7
16952024-02-27T21:44:37.469Z
16962024-02-27T21:44:37.469ZVerifying data now
16972024-02-27T21:44:41.457ZNo change after verify
16982024-02-27T21:44:41.458ZLoop: 7 Downstairs dump after verify (and repair):
16992024-02-27T21:44:41.475ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17002024-02-27T21:44:41.475Z 0 000-019 10 10 10 18 18 18 F F F
17012024-02-27T21:44:41.476Z 1 020-039 8 8 8 16 16 16 F F F
17022024-02-27T21:44:41.476Z 2 040-059 12 12 12 27 27 27 F F F
17032024-02-27T21:44:41.476Z 3 060-079 13 13 13 28 28 28 F F F
17042024-02-27T21:44:41.476Z 4 080-099 15 15 15 33 33 33 F F F
17052024-02-27T21:44:41.476Z 5 100-119 8 8 8 16 16 16 F F F
17062024-02-27T21:44:41.476Z 6 120-139 8 8 8 16 16 16 F F F
17072024-02-27T21:44:41.480Z 7 140-159 11 11 11 21 21 21 F F F
17082024-02-27T21:44:41.480Z 8 160-179 12 12 12 27 27 27 F F F
17092024-02-27T21:44:41.481Z 9 180-199 8 8 8 16 16 16 F F F
17102024-02-27T21:44:41.481Z 10 200-219 14 14 14 30 30 30 F F F
17112024-02-27T21:44:41.481Z 11 220-239 14 14 14 31 31 31 F F F
17122024-02-27T21:44:41.481Z 12 240-259 13 13 13 28 28 28 F F F
17132024-02-27T21:44:41.481Z 13 260-279 11 11 11 21 21 21 F F F
17142024-02-27T21:44:41.481Z 14 280-299 12 12 12 22 22 22 F F F
17152024-02-27T21:44:41.481Z 15 300-319 13 13 13 28 28 28 F F F
17162024-02-27T21:44:41.481Z 16 320-339 14 14 14 30 30 30 F F F
17172024-02-27T21:44:41.481Z 17 340-359 13 13 13 28 28 28 F F F
17182024-02-27T21:44:41.481Z 18 360-379 8 8 8 16 16 16 F F F
17192024-02-27T21:44:41.481Z 19 380-399 8 8 8 16 16 16 F F F
17202024-02-27T21:44:41.481Z 20 400-419 10 10 10 20 20 20 F F F
17212024-02-27T21:44:41.481Z 21 420-439 15 15 15 33 33 33 F F F
17222024-02-27T21:44:41.481Z 22 440-459 12 12 12 26 26 26 F F F
17232024-02-27T21:44:41.481Z 23 460-479 8 8 8 16 16 16 F F F
17242024-02-27T21:44:41.481Z 24 480-499 6 6 6 12 12 12 F F F
17252024-02-27T21:44:41.481Z 25 500-519 14 14 14 32 32 32 F F F
17262024-02-27T21:44:41.481Z 26 520-539 11 11 11 21 21 21 F F F
17272024-02-27T21:44:41.481Z 27 540-559 14 14 14 29 29 29 F F F
17282024-02-27T21:44:41.481Z 28 560-579 14 14 14 30 30 30 F F F
17292024-02-27T21:44:41.481Z 29 580-599 13 13 13 28 28 28 F F F
17302024-02-27T21:44:41.481ZMax gen: 15, Max flush: 33
17312024-02-27T21:44:41.482Z
17322024-02-27T21:44:41.482ZBegin loop 8
17332024-02-27T21:44:41.482ZDownstairs to restart: 0
17342024-02-27T21:44:41.493Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.905267281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202}
17352024-02-27T21:44:41.544Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.905550958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202}
17362024-02-27T21:44:41.545Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"Crucible runtime is spawned
17372024-02-27T21:44:41.545Zname":"crucible","level":30,"time":"2024-02-27T21:44:39.905601682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202}
17382024-02-27T21:44:41.545Z{"msg":"Crucible 0f1a94bc-47ad-4b18-9028-36797e40ce27 has session id: c24967df-5a10-4713-aef4-4c114a4f41b5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.905643697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17392024-02-27T21:44:41.545Z{"msg":"Upstairs opts: Upstairs UUID: 0f1a94bc-47ad-4b18-9028-36797e40ce27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.90569753Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17402024-02-27T21:44:41.545Z{"msg":"Crucible stats registered with UUID: 0f1a94bc-47ad-4b18-9028-36797e40ce27","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.905740255Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17412024-02-27T21:44:41.545Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.905770392Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202}
17422024-02-27T21:44:41.545Z{"msg":"0f1a94bc-47ad-4b18-9028-36797e40ce27 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.905797059Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17432024-02-27T21:44:41.545Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.905921204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17442024-02-27T21:44:41.546Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.905973548Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17452024-02-27T21:44:41.546Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906008254Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17462024-02-27T21:44:41.546Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906053828Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17472024-02-27T21:44:41.546Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906166625Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17482024-02-27T21:44:41.546Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906230867Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17492024-02-27T21:44:41.546Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906269373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"io task","client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17502024-02-27T21:44:41.546Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906300409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"io task","client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17512024-02-27T21:44:41.546Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906316068Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"io task","client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17522024-02-27T21:44:41.546Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906336785Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"io task","client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17532024-02-27T21:44:41.546Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906350893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"io task","client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17542024-02-27T21:44:41.547Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906364392Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"io task","client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17552024-02-27T21:44:41.547Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906842395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17562024-02-27T21:44:41.547Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.90688878Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17572024-02-27T21:44:41.547Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.906931865Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17582024-02-27T21:44:41.547Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.90697712Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17592024-02-27T21:44:41.547Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.907031883Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17602024-02-27T21:44:41.547Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.907050281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17612024-02-27T21:44:41.547Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.907477221Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17622024-02-27T21:44:41.547Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.907525545Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17632024-02-27T21:44:41.547Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.907556751Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17642024-02-27T21:44:41.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908060052Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17652024-02-27T21:44:41.548Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908109566Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17662024-02-27T21:44:41.548Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908137283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17672024-02-27T21:44:41.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908168069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17682024-02-27T21:44:41.548Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908200555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17692024-02-27T21:44:41.548Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908222573Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17702024-02-27T21:44:41.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908252089Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17712024-02-27T21:44:41.549Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908277486Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17722024-02-27T21:44:41.549Z{"msg":"[0]R flush_numbers[0..12]: [18, 16, 27, 28, 33, 16, 16, 21, 27, 16, 30, 31]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
17732024-02-27T21:44:41.549Z,"time":"2024-02-27T21:44:39.908309733Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17742024-02-27T21:44:41.549Z{"msg":"[0]R generation[0..12]: [10, 8, 12, 13, 15, 8, 8, 11, 12, 8, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.90833547Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17752024-02-27T21:44:41.549Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908362216Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17762024-02-27T21:44:41.549Z{"msg":"[1]R flush_numbers[0..12]: [18, 16, 27, 28, 33, 16, 16, 21, 27, 16, 30, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908386974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17772024-02-27T21:44:41.549Z{"msg":"[1]R generation[0..12]: [10, 8, 12, 13, 15, 8, 8, 11, 12, 8, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908406161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17782024-02-27T21:44:41.549Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908423009Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17792024-02-27T21:44:41.549Z{"msg":"[2]R flush_numbers[0..12]: [18, 16, 27, 28, 33, 16, 16, 21, 27, 16, 30, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908449176Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17802024-02-27T21:44:41.550Z{"msg":"[2]R generation[0..12]: [10, 8, 12, 13, 15, 8, 8, 11, 12, 8, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908467004Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17812024-02-27T21:44:41.550Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908491001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17822024-02-27T21:44:41.550Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.90850483Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17832024-02-27T21:44:41.550Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908518978Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17842024-02-27T21:44:41.550Z{"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908532196Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17852024-02-27T21:44:41.550Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908553914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"":"downstairs","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17862024-02-27T21:44:41.550Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908567712Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17872024-02-27T21:44:41.550Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.9085875Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"0","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17882024-02-27T21:44:41.551Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908602368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"1","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17892024-02-27T21:44:41.551Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908616677Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"client":"2","":"downstairs","session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17902024-02-27T21:44:41.551Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908630325Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17912024-02-27T21:44:41.551Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908643283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17922024-02-27T21:44:41.551Z{"msg":"0f1a94bc-47ad-4b18-9028-36797e40ce27 is now active with session: c24967df-5a10-4713-aef4-4c114a4f41b5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908663521Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17932024-02-27T21:44:41.551Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908675959Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202,"session_id":"c24967df-5a10-4713-aef4-4c114a4f41b5"}
17942024-02-27T21:44:41.551ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17952024-02-27T21:44:41.551Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:39.908703686Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202}
17962024-02-27T21:44:41.551ZLoading write count information from file "/tmp/test_repair_verify.data"
17972024-02-27T21:44:41.551ZRun Repair workload
17982024-02-27T21:44:41.551Z01/30 Write block 20 len 4096 data: 2
17992024-02-27T21:44:41.972Z02/30 Write block 415 len 20480 data: 2 2 2 2 2
18002024-02-27T21:44:41.972Z03/30 Read block 552 len 4096
18012024-02-27T21:44:41.972Z04/30 Write block 30 len 4096 data: 3
18022024-02-27T21:44:41.972Z05/30 Read block 123 len 12288
18032024-02-27T21:44:41.972Z06/30 Write block 310 len 20480 data: 5 5 5 4 4
18042024-02-27T21:44:41.972Z07/30 Read block 291 len 20480
18052024-02-27T21:44:41.972Z08/30 Read block 76 len 32768
18062024-02-27T21:44:41.972Z09/30 Write block 120 len 36864 data: 4 4 4 3 2 2 2 2 2
18072024-02-27T21:44:41.972Z10/30 Write block 295 len 40960 data: 2 2 2 2 2 2 2 2 2 2
18082024-02-27T21:44:41.972Z11/30 Read block 143 len 16384
18092024-02-27T21:44:41.972Z12/30 Read block 111 len 4096
18102024-02-27T21:44:41.972Z13/30 Read block 578 len 20480
18112024-02-27T21:44:41.972Z14/30 Read block 302 len 32768
18122024-02-27T21:44:41.972Z15/30 Write block 228 len 32768 data: 4 4 5 5 5 5 4 4
18132024-02-27T21:44:41.972Z16/30 Write block 335 len 32768 data: 4 3 4 3 4 3 4 4
18142024-02-27T21:44:41.972Z17/30 Read block 389 len 8192
18152024-02-27T21:44:41.973Z18/30 Read block 299 len 36864
18162024-02-27T21:44:41.973Z19/30 Read block 103 len 28672
18172024-02-27T21:44:41.973Z20/30 Write block 222 len 20480 data: 4 3 3 2 3
18182024-02-27T21:44:41.973Z21/30 Read block 195 len 12288
18192024-02-27T21:44:41.973Z22/30 Read block 286 len 16384
18202024-02-27T21:44:41.973Z23/30 Write block 370 len 8192 data: 2 2
18212024-02-27T21:44:41.973Z24/30 Read block 482 len 4096
18222024-02-27T21:44:41.973Z25/30 Write block 41 len 4096 data: 4
18232024-02-27T21:44:41.973Z26/30 Read block 189 len 20480
18242024-02-27T21:44:41.973Z27/30 Read block 443 len 16384
18252024-02-27T21:44:41.973Z28/30 Write block 231 len 40960 data: 6 6 6 5 5 3 2 2 2 2
18262024-02-27T21:44:41.974Z29/30 Write block 389 len 20480 data: 5 4 4 3 3
18272024-02-27T21:44:41.974Z30/30 Write block 256 len 4096 data: 3
18282024-02-27T21:44:41.974Z----------------------------------------------------------------
18292024-02-27T21:44:41.974Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18302024-02-27T21:44:41.974ZStates: Active Active Active
18312024-02-27T21:44:41.974ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18322024-02-27T21:44:41.974Z 1 Acked 1000 Write 1 Sent Done Done false
18332024-02-27T21:44:41.974Z 2 Acked 1001 Write 5 Sent Done Done false
18342024-02-27T21:44:41.974Z 3 Acked 1002 Read 1 Sent Done Done false
18352024-02-27T21:44:41.974Z 4 Acked 1003 Write 1 Sent Done Done false
18362024-02-27T21:44:41.974Z 5 Acked 1004 Read 3 Sent Done Done false
18372024-02-27T21:44:41.974Z 6 Acked 1005 Write 5 Sent Done Done false
18382024-02-27T21:44:41.975Z 7 Acked 1006 Read 5 Sent Done Done false
18392024-02-27T21:44:41.975Z 8 Acked 1007 Read 8 Sent Done Done false
18402024-02-27T21:44:41.975Z 9 Acked 1008 Write 9 Sent Done Done false
18412024-02-27T21:44:41.975Z 10 Acked 1009 Write 10 Sent Done Done false
18422024-02-27T21:44:41.975Z 11 Acked 1010 Read 4 Sent Done Done false
18432024-02-27T21:44:41.975Z 12 Acked 1011 Read 1 Sent Done Done false
18442024-02-27T21:44:41.975Z 13 Acked 1012 Read 5 Sent Done Done false
18452024-02-27T21:44:41.975Z 14 Acked 1013 Read 8 Sent Done Done false
18462024-02-27T21:44:41.975Z 15 Acked 1014 Write 8 Sent Done Done false
18472024-02-27T21:44:41.975Z 16 Acked 1015 Write 8 Sent Done Done false
18482024-02-27T21:44:41.975Z 17 Acked 1016 Read 2 Sent Done Done false
18492024-02-27T21:44:41.975Z 18 Acked 1017 Read 9 Sent Done Done false
18502024-02-27T21:44:41.976Z 19 Acked 1018 Read 7 Sent Done Done false
18512024-02-27T21:44:41.976Z 20 Acked 1019 Write 5 Sent Done Done false
18522024-02-27T21:44:41.976Z 21 Acked 1020 Read 3 Sent Done Done false
18532024-02-27T21:44:41.976Z 22 Acked 1021 Read 4 Sent Done Done false
18542024-02-27T21:44:41.976Z 23 Acked 1022 Write 2 Sent Done Done false
18552024-02-27T21:44:41.976Z 24 Acked 1023 Read 1 Sent Done Done false
18562024-02-27T21:44:41.976Z 25 Acked 1024 Write 1 Sent Done Done false
18572024-02-27T21:44:41.976Z 26 Acked 1025 Read 5 Sent Done Done false
18582024-02-27T21:44:41.976Z 27 Acked 1026 Read 4 Sent Done Done false
18592024-02-27T21:44:41.976Z 28 Acked 1027 Write 10 Sent Done Done false
18602024-02-27T21:44:41.976Z 29 Acked 1028 Write 5 Sent Sent Sent false
18612024-02-27T21:44:41.976Z 30 Acked 1029 Write 1 Sent Sent Sent false
18622024-02-27T21:44:41.976Z STATES DS:0 DS:1 DS:2 TOTAL
18632024-02-27T21:44:41.977Z New 0 0 0 0
18642024-02-27T21:44:41.977Z Sent 30 2 2 34
18652024-02-27T21:44:41.977Z Done 0 28 28 56
18662024-02-27T21:44:41.977Z Skipped 0 0 0 0
18672024-02-27T21:44:41.977Z Error 0 0 0 0
18682024-02-27T21:44:41.977ZLast Flush: 0 0 0
18692024-02-27T21:44:41.977ZDownstairs last five completed:
18702024-02-27T21:44:41.977ZUpstairs last five completed: 30 29 28 27 26
18712024-02-27T21:44:41.977Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:39.933680564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1202}
18722024-02-27T21:44:41.977ZWrote out file "/tmp/test_repair_verify.data"
18732024-02-27T21:44:41.977Z
18742024-02-27T21:44:41.977ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18752024-02-27T21:44:41.977ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18762024-02-27T21:44:41.978Z 0 000-019 10 10 10 18 18 18 F F F
18772024-02-27T21:44:41.978Z 1 020-039 8 8 8 16 16 16 F T T <---
18782024-02-27T21:44:41.978Z 2 040-059 12 12 12 27 27 27 F T T <---
18792024-02-27T21:44:41.978Z 3 060-079 13 13 13 28 28 28 F F F
18802024-02-27T21:44:41.978Z 4 080-099 15 15 15 33 33 33 F F F
18812024-02-27T21:44:41.978Z 5 100-119 8 8 8 16 16 16 F F F
18822024-02-27T21:44:41.978Z 6 120-139 8 8 8 16 16 16 F T T <---
18832024-02-27T21:44:41.978Z 7 140-159 11 11 11 21 21 21 F F F
18842024-02-27T21:44:41.978Z 8 160-179 12 12 12 27 27 27 F F F
18852024-02-27T21:44:41.978Z 9 180-199 8 8 8 16 16 16 F F F
18862024-02-27T21:44:41.978Z 10 200-219 14 14 14 30 30 30 F F F
18872024-02-27T21:44:41.978Z 11 220-239 14 14 14 31 31 31 F T T <---
18882024-02-27T21:44:41.978Z 12 240-259 13 13 13 28 28 28 F T T <---
18892024-02-27T21:44:41.979Z 13 260-279 11 11 11 21 21 21 F F F
18902024-02-27T21:44:41.979Z 14 280-299 12 12 12 22 22 22 F T T <---
18912024-02-27T21:44:41.979Z 15 300-319 13 13 13 28 28 28 F T T <---
18922024-02-27T21:44:41.979Z 16 320-339 14 14 14 30 30 30 F T T <---
18932024-02-27T21:44:41.979Z 17 340-359 13 13 13 28 28 28 F T T <---
18942024-02-27T21:44:41.979Z 18 360-379 8 8 8 16 16 16 F T T <---
18952024-02-27T21:44:41.979Z 19 380-399 8 8 8 16 16 16 F T T <---
18962024-02-27T21:44:41.979Z 20 400-419 10 10 10 20 20 20 F T T <---
18972024-02-27T21:44:41.979Z 21 420-439 15 15 15 33 33 33 F F F
18982024-02-27T21:44:41.979Z 22 440-459 12 12 12 26 26 26 F F F
18992024-02-27T21:44:41.986Z 23 460-479 8 8 8 16 16 16 F F F
19002024-02-27T21:44:41.986Z 24 480-499 6 6 6 12 12 12 F F F
19012024-02-27T21:44:41.986Z 25 500-519 14 14 14 32 32 32 F F F
19022024-02-27T21:44:41.986Z 26 520-539 11 11 11 21 21 21 F F F
19032024-02-27T21:44:41.986Z 27 540-559 14 14 14 29 29 29 F F F
19042024-02-27T21:44:41.986Z 28 560-579 14 14 14 30 30 30 F F F
19052024-02-27T21:44:41.986Z 29 580-599 13 13 13 28 28 28 F F F
19062024-02-27T21:44:41.986ZMax gen: 15, Max flush: 33
19072024-02-27T21:44:41.987ZError: Difference in extent metadata found!
19082024-02-27T21:44:41.987ZA Difference in extent metadata is expected here
19092024-02-27T21:44:41.987ZOn loop 8
19102024-02-27T21:44:41.987Z
19112024-02-27T21:44:41.987ZVerifying data now
19122024-02-27T21:44:46.177ZNo change after verify
19132024-02-27T21:44:46.177ZLoop: 8 Downstairs dump after verify (and repair):
19142024-02-27T21:44:46.186ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19152024-02-27T21:44:46.186Z 0 000-019 10 10 10 18 18 18 F F F
19162024-02-27T21:44:46.186Z 1 020-039 16 16 16 34 34 34 F F F
19172024-02-27T21:44:46.186Z 2 040-059 16 16 16 34 34 34 F F F
19182024-02-27T21:44:46.186Z 3 060-079 13 13 13 28 28 28 F F F
19192024-02-27T21:44:46.186Z 4 080-099 15 15 15 33 33 33 F F F
19202024-02-27T21:44:46.191Z 5 100-119 8 8 8 16 16 16 F F F
19212024-02-27T21:44:46.191Z 6 120-139 16 16 16 34 34 34 F F F
19222024-02-27T21:44:46.191Z 7 140-159 11 11 11 21 21 21 F F F
19232024-02-27T21:44:46.192Z 8 160-179 12 12 12 27 27 27 F F F
19242024-02-27T21:44:46.192Z 9 180-199 8 8 8 16 16 16 F F F
19252024-02-27T21:44:46.192Z 10 200-219 14 14 14 30 30 30 F F F
19262024-02-27T21:44:46.192Z 11 220-239 16 16 16 34 34 34 F F F
19272024-02-27T21:44:46.193Z 12 240-259 16 16 16 34 34 34 F F F
19282024-02-27T21:44:46.193Z 13 260-279 11 11 11 21 21 21 F F F
19292024-02-27T21:44:46.193Z 14 280-299 16 16 16 34 34 34 F F F
19302024-02-27T21:44:46.193Z 15 300-319 16 16 16 34 34 34 F F F
19312024-02-27T21:44:46.194Z 16 320-339 16 16 16 34 34 34 F F F
19322024-02-27T21:44:46.194Z 17 340-359 16 16 16 34 34 34 F F F
19332024-02-27T21:44:46.194Z 18 360-379 16 16 16 34 34 34 F F F
19342024-02-27T21:44:46.194Z 19 380-399 16 16 16 34 34 34 F F F
19352024-02-27T21:44:46.194Z 20 400-419 16 16 16 34 34 34 F F F
19362024-02-27T21:44:46.194Z 21 420-439 15 15 15 33 33 33 F F F
19372024-02-27T21:44:46.195Z 22 440-459 12 12 12 26 26 26 F F F
19382024-02-27T21:44:46.195Z 23 460-479 8 8 8 16 16 16 F F F
19392024-02-27T21:44:46.195Z 24 480-499 6 6 6 12 12 12 F F F
19402024-02-27T21:44:46.195Z 25 500-519 14 14 14 32 32 32 F F F
19412024-02-27T21:44:46.195Z 26 520-539 11 11 11 21 21 21 F F F
19422024-02-27T21:44:46.195Z 27 540-559 14 14 14 29 29 29 F F F
19432024-02-27T21:44:46.195Z 28 560-579 14 14 14 30 30 30 F F F
19442024-02-27T21:44:46.196Z 29 580-599 13 13 13 28 28 28 F F F
19452024-02-27T21:44:46.196ZMax gen: 16, Max flush: 34
19462024-02-27T21:44:46.196Z
19472024-02-27T21:44:46.196ZBegin loop 9
19482024-02-27T21:44:46.196ZDownstairs to restart: 0
19492024-02-27T21:44:46.209Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.62102954Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210}
19502024-02-27T21:44:46.209Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621341673Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210}
19512024-02-27T21:44:46.222Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621374639Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210}
19522024-02-27T21:44:46.222Z{"msg":"Crucible runtime is spawned
19532024-02-27T21:44:46.222ZCrucible b4b19cd8-958b-425b-947c-574ed5015cb6 has session id: ef391480-0221-4f5c-9258-426caeedb5dd","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.62145471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19542024-02-27T21:44:46.222Z{"msg":"Upstairs opts: Upstairs UUID: b4b19cd8-958b-425b-947c-574ed5015cb6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621502294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19552024-02-27T21:44:46.222Z{"msg":"Crucible stats registered with UUID: b4b19cd8-958b-425b-947c-574ed5015cb6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621551749Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19562024-02-27T21:44:46.223Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621601943Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210}
19572024-02-27T21:44:46.223Z{"msg":"b4b19cd8-958b-425b-947c-574ed5015cb6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621672414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19582024-02-27T21:44:46.223Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.62170947Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19592024-02-27T21:44:46.223Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621737007Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19602024-02-27T21:44:46.223Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621766583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19612024-02-27T21:44:46.223Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.62179512Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19622024-02-27T21:44:46.223Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621831045Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19632024-02-27T21:44:46.223Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621849074Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19642024-02-27T21:44:46.224Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.621953731Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"io task","client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19652024-02-27T21:44:46.224Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.622002895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"io task","client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19662024-02-27T21:44:46.224Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.622032942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"io task","client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19672024-02-27T21:44:46.224Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.622060498Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"io task","client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19682024-02-27T21:44:46.224Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.622085946Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"io task","client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19692024-02-27T21:44:46.224Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.622197042Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"io task","client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19702024-02-27T21:44:46.224Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.622872323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19712024-02-27T21:44:46.224Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.622916418Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19722024-02-27T21:44:46.224Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.622993109Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19732024-02-27T21:44:46.225Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.623047592Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19742024-02-27T21:44:46.225Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.623077949Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19752024-02-27T21:44:46.225Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.623124083Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19762024-02-27T21:44:46.225Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.623516187Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19772024-02-27T21:44:46.225Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.62357858Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19782024-02-27T21:44:46.226Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.623632443Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19792024-02-27T21:44:46.226Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624090399Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19802024-02-27T21:44:46.226Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624145843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19812024-02-27T21:44:46.226Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.62417347Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19822024-02-27T21:44:46.226Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624259419Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19832024-02-27T21:44:46.226Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624290516Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19842024-02-27T21:44:46.227Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624304954Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19852024-02-27T21:44:46.227Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624324062Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19862024-02-27T21:44:46.227Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.62433766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19872024-02-27T21:44:46.227Z{"msg":"[0]R flush_numbers[0..12]: [18, 34, 34, 28, 33, 16, 34, 21, 27, 16, 30, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624358128Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19882024-02-27T21:44:46.227Z{"msg":"[0]R generation[0..12]: [10, 16, 16, 13, 15, 8, 16, 11, 12, 8, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624372356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19892024-02-27T21:44:46.228Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624393494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19902024-02-27T21:44:46.228Z{"msg":"[1]R flush_numbers[0..12]: [18, 34, 34, 28, 33, 16, 34, 21, 27, 16, 30, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624408632Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19912024-02-27T21:44:46.228Z{"msg":"[1]R generation[0..12]: [10, 16, 16, 13, 15, 8, 16, 11, 12, 8, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624442708Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19922024-02-27T21:44:46.228Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624466195Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19932024-02-27T21:44:46.228Z{"msg":"[2]R flush_numbers[0..12]: [18, 34, 34, 28, 33, 16, 34, 21, 27, 16, 30, 34]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
19942024-02-27T21:44:46.228Z,"time":"2024-02-27T21:44:44.624490282Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19952024-02-27T21:44:46.228Z{"msg":"[2]R generation[0..12]: [10, 16, 16, 13, 15, 8, 16, 11, 12, 8, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624505281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19962024-02-27T21:44:46.229Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624526198Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19972024-02-27T21:44:46.229Z{"msg":"Max found gen is 17","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624539846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19982024-02-27T21:44:46.229Z{"msg":"Generation requested: 18 >= found:17","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624560114Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
19992024-02-27T21:44:46.229Z{"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624574013Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20002024-02-27T21:44:46.229Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624603029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"":"downstairs","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20012024-02-27T21:44:46.229Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624630896Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20022024-02-27T21:44:46.229Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624646564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"0","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20032024-02-27T21:44:46.229Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624672551Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"1","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20042024-02-27T21:44:46.230Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624693708Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"client":"2","":"downstairs","session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20052024-02-27T21:44:46.230Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624726515Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20062024-02-27T21:44:46.230Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.62476366Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20072024-02-27T21:44:46.230Z{"msg":"b4b19cd8-958b-425b-947c-574ed5015cb6 is now active with session: ef391480-0221-4f5c-9258-426caeedb5dd","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624792037Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20082024-02-27T21:44:46.230Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624805275Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210,"session_id":"ef391480-0221-4f5c-9258-426caeedb5dd"}
20092024-02-27T21:44:46.230Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:44.624837971Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210}
20102024-02-27T21:44:46.230ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20112024-02-27T21:44:46.230ZLoading write count information from file "/tmp/test_repair_verify.data"
20122024-02-27T21:44:46.231ZRun Repair workload
20132024-02-27T21:44:46.231Z01/30 Write block 512 len 24576 data: 3 3 3 3 4 4
20142024-02-27T21:44:46.231Z02/30 Write block 67 len 8192 data: 2 2
20152024-02-27T21:44:46.231Z03/30 Read block 154 len 40960
20162024-02-27T21:44:46.238Z04/30 Write block 260 len 12288 data: 3 2 2
20172024-02-27T21:44:46.238Z05/30 Read block 61 len 24576
20182024-02-27T21:44:46.238Z06/30 Read block 139 len 8192
20192024-02-27T21:44:46.238Z07/30 Write block 49 len 24576 data: 2 2 2 2 2 2
20202024-02-27T21:44:46.238Z08/30 Write block 199 len 4096 data: 2
20212024-02-27T21:44:46.238Z09/30 Read block 67 len 4096
20222024-02-27T21:44:46.239Z10/30 Flush
20232024-02-27T21:44:46.239Z11/30 Write block 557 len 28672 data: 5 4 3 3 4 4 3
20242024-02-27T21:44:46.239Z12/30 Read block 94 len 28672
20252024-02-27T21:44:46.239Z13/30 Write block 426 len 36864 data: 2 2 2 3 3 3 3 3 3
20262024-02-27T21:44:46.239Z14/30 Flush
20272024-02-27T21:44:46.239Z15/30 Write block 93 len 20480 data: 2 2 2 2 2
20282024-02-27T21:44:46.239Z16/30 Read block 80 len 8192
20292024-02-27T21:44:46.239Z17/30 Read block 549 len 28672
20302024-02-27T21:44:46.239Z18/30 Read block 562 len 12288
20312024-02-27T21:44:46.239Z19/30 Read block 318 len 40960
20322024-02-27T21:44:46.239Z20/30 Flush
20332024-02-27T21:44:46.239Z21/30 Write block 302 len 32768 data: 3 3 3 2 3 4 4 5
20342024-02-27T21:44:46.240Z22/30 Flush
20352024-02-27T21:44:46.278Z23/30 Write block 222 len 12288 data: 5 4 4
20362024-02-27T21:44:46.278Z24/30 Write block 417 len 16384 data: 3 3 3 2
20372024-02-27T21:44:46.279Z25/30 Write block 348 len 12288 data: 2 2 2
20382024-02-27T21:44:46.279Z26/30 Write block 74 len 4096 data: 4
20392024-02-27T21:44:46.279Z27/30 Write block 499 len 8192 data: 4 3
20402024-02-27T21:44:46.279Z28/30 Read block 134 len 24576
20412024-02-27T21:44:46.280Z29/30 Read block 385 len 32768
20422024-02-27T21:44:46.280Z30/30 Read block 365 len 8192
20432024-02-27T21:44:46.280Z----------------------------------------------------------------
20442024-02-27T21:44:46.281Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20452024-02-27T21:44:46.281ZStates: Active Active Active
20462024-02-27T21:44:46.281ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20472024-02-27T21:44:46.281Z 1 Acked 1000 Write 6 Done Done Done false
20482024-02-27T21:44:46.282Z 2 Acked 1001 Write 2 Done Done Done false
20492024-02-27T21:44:46.282Z 3 Acked 1002 Read 10 Done Done Done false
20502024-02-27T21:44:46.282Z 4 Acked 1003 Write 3 Done Done Done false
20512024-02-27T21:44:46.282Z 5 Acked 1004 Read 6 Done Done Done false
20522024-02-27T21:44:46.283Z 6 Acked 1005 Read 2 Done Done Done false
20532024-02-27T21:44:46.283Z 7 Acked 1006 Write 6 Sent Done Done false
20542024-02-27T21:44:46.283Z 8 Acked 1007 Write 1 Sent Done Done false
20552024-02-27T21:44:46.283Z 9 Acked 1008 Read 1 Sent Done Done false
20562024-02-27T21:44:46.284Z 10 Acked 1009 Flush 0 Sent Done Done false
20572024-02-27T21:44:46.284Z 11 Acked 1010 Write 7 Sent Done Done false
20582024-02-27T21:44:46.284Z 12 Acked 1011 Read 7 Sent Done Done false
20592024-02-27T21:44:46.284Z 13 Acked 1012 Write 9 Sent Done Done false
20602024-02-27T21:44:46.284Z 14 Acked 1013 Flush 0 Sent Done Done false
20612024-02-27T21:44:46.285Z 15 Acked 1014 Write 5 Sent Done Done false
20622024-02-27T21:44:46.285Z 16 Acked 1015 Read 2 Sent Done Done false
20632024-02-27T21:44:46.285Z 17 Acked 1016 Read 7 Sent Done Done false
20642024-02-27T21:44:46.285Z 18 Acked 1017 Read 3 Sent Done Done false
20652024-02-27T21:44:46.285Z 19 Acked 1018 Read 10 Sent Done Done false
20662024-02-27T21:44:46.286Z 20 Acked 1019 Flush 0 Sent Done Done false
20672024-02-27T21:44:46.286Z 21 Acked 1020 Write 8 Sent Done Done false
20682024-02-27T21:44:46.286Z 22 Acked 1021 Flush 0 Sent Done Done false
20692024-02-27T21:44:46.286Z 23 Acked 1022 Write 3 Sent Done Done false
20702024-02-27T21:44:46.286Z 24 Acked 1023 Write 4 Sent Done Done false
20712024-02-27T21:44:46.286Z 25 Acked 1024 Write 3 Sent Done Done false
20722024-02-27T21:44:46.286Z 26 Acked 1025 Write 1 Sent Done Done false
20732024-02-27T21:44:46.286Z 27 Acked 1026 Write 2 Sent Done Done false
20742024-02-27T21:44:46.287Z 28 Acked 1027 Read 6 Sent Done Done false
20752024-02-27T21:44:46.287Z 29 Acked 1028 Read 8 Sent Done Done false
20762024-02-27T21:44:46.287Z 30 Acked 1029 Read 2 Sent Done Done false
20772024-02-27T21:44:46.287Z STATES DS:0 DS:1 DS:2 TOTAL
20782024-02-27T21:44:46.287Z New 0 0 0 0
20792024-02-27T21:44:46.287Z Sent 24 0 0 24
20802024-02-27T21:44:46.287Z Done 6 30 30 66
20812024-02-27T21:44:46.288Z Skipped 0 0 0 0
20822024-02-27T21:44:46.288Z Error 0 0 0 0
20832024-02-27T21:44:46.288ZLast Flush: 0 1021 1021
20842024-02-27T21:44:46.288ZDownstairs last five completed:
20852024-02-27T21:44:46.288ZUpstairs last five completed: 30 29 28 27 26
20862024-02-27T21:44:46.288Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:44.655798555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1210}
20872024-02-27T21:44:46.288ZWrote out file "/tmp/test_repair_verify.data"
20882024-02-27T21:44:46.289Z
20892024-02-27T21:44:46.289ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20902024-02-27T21:44:46.315ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20912024-02-27T21:44:46.315Z 0 000-019 10 10 10 18 18 18 F F F
20922024-02-27T21:44:46.316Z 1 020-039 16 16 16 34 34 34 F F F
20932024-02-27T21:44:46.316Z 2 040-059 16 18 18 34 35 35 F F F <---
20942024-02-27T21:44:46.316Z 3 060-079 13 18 18 28 35 35 T T T <---
20952024-02-27T21:44:46.316Z 4 080-099 15 18 18 33 37 37 F F F <---
20962024-02-27T21:44:46.316Z 5 100-119 8 8 8 16 16 16 F F F
20972024-02-27T21:44:46.316Z 6 120-139 16 16 16 34 34 34 F F F
20982024-02-27T21:44:46.316Z 7 140-159 11 11 11 21 21 21 F F F
20992024-02-27T21:44:46.316Z 8 160-179 12 12 12 27 27 27 F F F
21002024-02-27T21:44:46.317Z 9 180-199 8 18 18 16 35 35 F F F <---
21012024-02-27T21:44:46.317Z 10 200-219 14 14 14 30 30 30 F F F
21022024-02-27T21:44:46.317Z 11 220-239 16 16 16 34 34 34 F T T <---
21032024-02-27T21:44:46.317Z 12 240-259 16 16 16 34 34 34 F F F
21042024-02-27T21:44:46.317Z 13 260-279 11 18 18 21 35 35 T F F <---
21052024-02-27T21:44:46.317Z 14 280-299 16 16 16 34 34 34 F F F
21062024-02-27T21:44:46.318Z 15 300-319 16 18 18 34 38 38 F F F <---
21072024-02-27T21:44:46.318Z 16 320-339 16 16 16 34 34 34 F F F
21082024-02-27T21:44:46.318Z 17 340-359 16 16 16 34 34 34 F T T <---
21092024-02-27T21:44:46.318Z 18 360-379 16 16 16 34 34 34 F F F
21102024-02-27T21:44:46.318Z 19 380-399 16 16 16 34 34 34 F F F
21112024-02-27T21:44:46.318Z 20 400-419 16 16 16 34 34 34 F T T <---
21122024-02-27T21:44:46.318Z 21 420-439 15 18 18 33 36 36 F T T <---
21132024-02-27T21:44:46.318Z 22 440-459 12 12 12 26 26 26 F F F
21142024-02-27T21:44:46.318Z 23 460-479 8 8 8 16 16 16 F F F
21152024-02-27T21:44:46.319Z 24 480-499 6 6 6 12 12 12 F T T <---
21162024-02-27T21:44:46.319Z 25 500-519 14 18 18 32 35 35 T T T <---
21172024-02-27T21:44:46.319Z 26 520-539 11 11 11 21 21 21 F F F
21182024-02-27T21:44:46.319Z 27 540-559 14 18 18 29 36 36 F F F <---
21192024-02-27T21:44:46.319Z 28 560-579 14 18 18 30 36 36 F F F <---
21202024-02-27T21:44:46.319Z 29 580-599 13 13 13 28 28 28 F F F
21212024-02-27T21:44:46.320ZMax gen: 18, Max flush: 38
21222024-02-27T21:44:46.320ZError: Difference in extent metadata found!
21232024-02-27T21:44:46.320ZA Difference in extent metadata is expected here
21242024-02-27T21:44:46.320ZOn loop 9
21252024-02-27T21:44:46.320Z
21262024-02-27T21:44:46.320ZVerifying data now
21272024-02-27T21:44:51.959ZNo change after verify
21282024-02-27T21:44:51.960ZLoop: 9 Downstairs dump after verify (and repair):
21292024-02-27T21:44:51.979ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21302024-02-27T21:44:51.979Z 0 000-019 10 10 10 18 18 18 F F F
21312024-02-27T21:44:51.979Z 1 020-039 16 16 16 34 34 34 F F F
21322024-02-27T21:44:51.979Z 2 040-059 18 18 18 35 35 35 F F F
21332024-02-27T21:44:51.979Z 3 060-079 19 19 19 39 39 39 F F F
21342024-02-27T21:44:51.979Z 4 080-099 18 18 18 37 37 37 F F F
21352024-02-27T21:44:51.979Z 5 100-119 8 8 8 16 16 16 F F F
21362024-02-27T21:44:51.979Z 6 120-139 16 16 16 34 34 34 F F F
21372024-02-27T21:44:51.979Z 7 140-159 11 11 11 21 21 21 F F F
21382024-02-27T21:44:52.044Z 8 160-179 12 12 12 27 27 27 F F F
21392024-02-27T21:44:52.044Z 9 180-199 18 18 18 35 35 35 F F F
21402024-02-27T21:44:52.044Z 10 200-219 14 14 14 30 30 30 F F F
21412024-02-27T21:44:52.044Z 11 220-239 19 19 19 39 39 39 F F F
21422024-02-27T21:44:52.044Z 12 240-259 16 16 16 34 34 34 F F F
21432024-02-27T21:44:52.045Z 13 260-279 18 18 18 35 35 35 F F F
21442024-02-27T21:44:52.045Z 14 280-299 16 16 16 34 34 34 F F F
21452024-02-27T21:44:52.045Z 15 300-319 18 18 18 38 38 38 F F F
21462024-02-27T21:44:52.045Z 16 320-339 16 16 16 34 34 34 F F F
21472024-02-27T21:44:52.045Z 17 340-359 19 19 19 39 39 39 F F F
21482024-02-27T21:44:52.045Z 18 360-379 16 16 16 34 34 34 F F F
21492024-02-27T21:44:52.045Z 19 380-399 16 16 16 34 34 34 F F F
21502024-02-27T21:44:52.045Z 20 400-419 19 19 19 39 39 39 F F F
21512024-02-27T21:44:52.046Z 21 420-439 19 19 19 39 39 39 F F F
21522024-02-27T21:44:52.046Z 22 440-459 12 12 12 26 26 26 F F F
21532024-02-27T21:44:52.046Z 23 460-479 8 8 8 16 16 16 F F F
21542024-02-27T21:44:52.046Z 24 480-499 19 19 19 39 39 39 F F F
21552024-02-27T21:44:52.046Z 25 500-519 19 19 19 39 39 39 F F F
21562024-02-27T21:44:52.046Z 26 520-539 11 11 11 21 21 21 F F F
21572024-02-27T21:44:52.046Z 27 540-559 18 18 18 36 36 36 F F F
21582024-02-27T21:44:52.046Z 28 560-579 18 18 18 36 36 36 F F F
21592024-02-27T21:44:52.046Z 29 580-599 13 13 13 28 28 28 F F F
21602024-02-27T21:44:52.046ZMax gen: 19, Max flush: 39
21612024-02-27T21:44:52.047Z
21622024-02-27T21:44:52.047ZBegin loop 10
21632024-02-27T21:44:52.047ZDownstairs to restart: 0
21642024-02-27T21:44:52.047Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.402537494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218}
21652024-02-27T21:44:52.047Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.402777766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218}
21662024-02-27T21:44:52.047Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.402796523Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218}
21672024-02-27T21:44:52.047Z{"msg":"Crucible 79e10052-8937-4dd6-89f9-13d6a3504a02 has session id: f0bed9c7-339d-4e53-9d07-e5c2cdbcb396","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.402810322Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21682024-02-27T21:44:52.047Z{"msg":"Upstairs opts: Upstairs UUID: 79e10052-8937-4dd6-89f9-13d6a3504a02, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.402844988Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21692024-02-27T21:44:52.047Z{"msg":"Crucible stats registered with UUID: 79e10052-8937-4dd6-89f9-13d6a3504a02","v":0,"name":"crucible"Crucible runtime is spawned
21702024-02-27T21:44:52.047Z,"level":30,"time":"2024-02-27T21:44:50.402887343Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21712024-02-27T21:44:52.048Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.402968263Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218}
21722024-02-27T21:44:52.048Z{"msg":"79e10052-8937-4dd6-89f9-13d6a3504a02 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403140323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21732024-02-27T21:44:52.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403180528Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21742024-02-27T21:44:52.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403216494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21752024-02-27T21:44:52.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403260119Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21762024-02-27T21:44:52.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403354178Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21772024-02-27T21:44:52.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403407581Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21782024-02-27T21:44:52.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403437388Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21792024-02-27T21:44:52.048Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403461915Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"io task","client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21802024-02-27T21:44:52.048Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403494901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"io task","client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21812024-02-27T21:44:52.049Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403522088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"io task","client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21822024-02-27T21:44:52.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403547135Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"io task","client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21832024-02-27T21:44:52.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403572822Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"io task","client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21842024-02-27T21:44:52.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.403603448Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"io task","client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21852024-02-27T21:44:52.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.404134336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21862024-02-27T21:44:52.621Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.404172141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21872024-02-27T21:44:52.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.40418816Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21882024-02-27T21:44:52.622Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.404222685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21892024-02-27T21:44:52.622Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.404294127Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21902024-02-27T21:44:52.622Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.404310595Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21912024-02-27T21:44:52.622Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.404724636Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21922024-02-27T21:44:52.622Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.404771921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21932024-02-27T21:44:52.622Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405001324Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21942024-02-27T21:44:52.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405254414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21952024-02-27T21:44:52.623Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.40528943Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21962024-02-27T21:44:52.623Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405304748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21972024-02-27T21:44:52.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405366491Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21982024-02-27T21:44:52.623Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405392547Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
21992024-02-27T21:44:52.624Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405407396Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22002024-02-27T21:44:52.624Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405495246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22012024-02-27T21:44:52.624Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.40554072Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22022024-02-27T21:44:52.624Z{"msg":"[0]R flush_numbers[0..12]: [18, 34, 35, 39, 37, 16, 34, 21, 27, 35, 30, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405555818Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22032024-02-27T21:44:52.624Z{"msg":"[0]R generation[0..12]: [10, 16, 18, 19, 18, 8, 16, 11, 12, 18, 14, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405577046Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22042024-02-27T21:44:52.625Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405592524Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22052024-02-27T21:44:52.625Z{"msg":"[1]R flush_numbers[0..12]: [18, 34, 35, 39, 37, 16, 34, 21, 27, 35, 30, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405614821Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22062024-02-27T21:44:52.625Z{"msg":"[1]R generation[0..12]: [10, 16, 18, 19, 18, 8, 16, 11, 12, 18, 14, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.40562909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22072024-02-27T21:44:52.625Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405650347Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22082024-02-27T21:44:52.625Z{"msg":"[2]R flush_numbers[0..12]: [18, 34, 35, 39, 37, 16, 34, 21, 27, 35, 30, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405664786Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22092024-02-27T21:44:52.626ZWait for a query_work_queue command to finish before sending IO
22102024-02-27T21:44:52.626Z{"msg":"[2]R generation[0..12]: [10, 16, 18, 19, 18, 8, 16, 11, 12, 18, 14, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405689732Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22112024-02-27T21:44:52.626Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405704291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22122024-02-27T21:44:52.626Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405717709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22132024-02-27T21:44:52.626Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405737947Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22142024-02-27T21:44:52.626Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405760314Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22152024-02-27T21:44:52.627Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405791851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"":"downstairs","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22162024-02-27T21:44:52.627Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405814558Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22172024-02-27T21:44:52.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405848314Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"0","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22182024-02-27T21:44:52.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405884909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"1","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22192024-02-27T21:44:52.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405919476Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"client":"2","":"downstairs","session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22202024-02-27T21:44:52.627Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405955721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22212024-02-27T21:44:52.627Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405980288Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22222024-02-27T21:44:52.627Z{"msg":"79e10052-8937-4dd6-89f9-13d6a3504a02 is now active with session: f0bed9c7-339d-4e53-9d07-e5c2cdbcb396","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.405995087Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22232024-02-27T21:44:52.627Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.406015614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218,"session_id":"f0bed9c7-339d-4e53-9d07-e5c2cdbcb396"}
22242024-02-27T21:44:52.627Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:50.406033702Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218}
22252024-02-27T21:44:52.628ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22262024-02-27T21:44:52.628ZLoading write count information from file "/tmp/test_repair_verify.data"
22272024-02-27T21:44:52.628ZRun Repair workload
22282024-02-27T21:44:52.628Z01/30 Write block 461 len 24576 data: 2 3 3 2 2 2
22292024-02-27T21:44:52.628Z02/30 Read block 200 len 20480
22302024-02-27T21:44:52.629Z03/30 Read block 345 len 24576
22312024-02-27T21:44:52.629Z04/30 Write block 335 len 16384 data: 5 4 5 4
22322024-02-27T21:44:52.629Z05/30 Write block 473 len 12288 data: 2 2 2
22332024-02-27T21:44:52.629Z06/30 Read block 555 len 32768
22342024-02-27T21:44:52.629Z07/30 Write block 509 len 24576 data: 2 3 3 4 4 4
22352024-02-27T21:44:52.630Z08/30 Read block 118 len 12288
22362024-02-27T21:44:52.630Z09/30 Read block 573 len 8192
22372024-02-27T21:44:52.630Z10/30 Write block 553 len 32768 data: 4 4 4 3 6 5 4 4
22382024-02-27T21:44:52.630Z11/30 Read block 388 len 20480
22392024-02-27T21:44:52.630Z12/30 Write block 417 len 12288 data: 4 4 4
22402024-02-27T21:44:52.631Z13/30 Read block 15 len 12288
22412024-02-27T21:44:52.631Z14/30 Write block 156 len 12288 data: 4 4 3
22422024-02-27T21:44:52.631Z15/30 Write block 307 len 20480 data: 5 5 6 6 6
22432024-02-27T21:44:52.631Z16/30 Write block 289 len 28672 data: 5 4 3 3 3 3 3
22442024-02-27T21:44:52.631Z17/30 Write block 443 len 36864 data: 3 3 3 4 4 4 3 3 3
22452024-02-27T21:44:52.632Z18/30 Flush
22462024-02-27T21:44:52.632Z19/30 Write block 155 len 20480 data: 4 5 5 4 3
22472024-02-27T21:44:52.632Z20/30 Flush
22482024-02-27T21:44:52.633Z21/30 Write block 224 len 36864 data: 5 3 4 5 5 5 6 7 7
22492024-02-27T21:44:52.633Z22/30 Flush
22502024-02-27T21:44:52.633Z23/30 Write block 511 len 28672 data: 4 5 5 5 4 5 5
22512024-02-27T21:44:52.633Z24/30 Read block 209 len 24576
22522024-02-27T21:44:52.633Z25/30 Flush
22532024-02-27T21:44:52.633Z26/30 Write block 412 len 32768 data: 3 3 3 3 3 5 5 5
22542024-02-27T21:44:52.634Z27/30 Write block 494 len 20480 data: 5 3 3 4 4
22552024-02-27T21:44:52.634Z28/30 Read block 152 len 28672
22562024-02-27T21:44:52.634Z29/30 Read block 471 len 28672
22572024-02-27T21:44:52.634Z30/30 Read block 328 len 4096
22582024-02-27T21:44:52.634Z----------------------------------------------------------------
22592024-02-27T21:44:52.635Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22602024-02-27T21:44:52.635ZStates: Active Active Active
22612024-02-27T21:44:52.635ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22622024-02-27T21:44:52.635Z 1 Acked 1000 Write 6 Done Done Done false
22632024-02-27T21:44:52.635Z 2 Acked 1001 Read 5 Done Done Done false
22642024-02-27T21:44:52.635Z 3 Acked 1002 Read 6 Done Done Done false
22652024-02-27T21:44:52.636Z 4 Acked 1003 Write 4 Sent Done Done false
22662024-02-27T21:44:52.636Z 5 Acked 1004 Write 3 Sent Done Done false
22672024-02-27T21:44:52.636Z 6 Acked 1005 Read 8 Sent Done Done false
22682024-02-27T21:44:52.636Z 7 Acked 1006 Write 6 Sent Done Done false
22692024-02-27T21:44:52.636Z 8 Acked 1007 Read 3 Sent Done Done false
22702024-02-27T21:44:52.636Z 9 Acked 1008 Read 2 Sent Done Done false
22712024-02-27T21:44:52.636Z 10 Acked 1009 Write 8 Sent Done Done false
22722024-02-27T21:44:52.637Z 11 Acked 1010 Read 5 Sent Done Done false
22732024-02-27T21:44:52.637Z 12 Acked 1011 Write 3 Sent Done Done false
22742024-02-27T21:44:52.637Z 13 Acked 1012 Read 3 Sent Done Done false
22752024-02-27T21:44:52.637Z 14 Acked 1013 Write 3 Sent Done Done false
22762024-02-27T21:44:52.637Z 15 Acked 1014 Write 5 Sent Done Done false
22772024-02-27T21:44:52.637Z 16 Acked 1015 Write 7 Sent Done Done false
22782024-02-27T21:44:52.637Z 17 Acked 1016 Write 9 Sent Done Done false
22792024-02-27T21:44:52.637Z 18 Acked 1017 Flush 0 Sent Done Done false
22802024-02-27T21:44:52.638Z 19 Acked 1018 Write 5 Sent Done Done false
22812024-02-27T21:44:52.638Z 20 Acked 1019 Flush 0 Sent Done Done false
22822024-02-27T21:44:52.638Z 21 Acked 1020 Write 9 Sent Done Done false
22832024-02-27T21:44:52.638Z 22 Acked 1021 Flush 0 Sent Done Done false
22842024-02-27T21:44:52.638Z 23 Acked 1022 Write 7 Sent Done Done false
22852024-02-27T21:44:52.673Z 24 Acked 1023 Read 6 Sent Done Done false
22862024-02-27T21:44:52.673Z 25 Acked 1024 Flush 0 Sent Done Done false
22872024-02-27T21:44:52.673Z 26 Acked 1025 Write 8 Sent Done Done false
22882024-02-27T21:44:52.673Z 27 Acked 1026 Write 5 Sent Done Done false
22892024-02-27T21:44:52.673Z 28 Acked 1027 Read 7 Sent Done Done false
22902024-02-27T21:44:52.673Z 29 Acked 1028 Read 7 Sent Done Done false
22912024-02-27T21:44:52.673Z 30 Acked 1029 Read 1 Sent Sent Done false
22922024-02-27T21:44:52.674Z STATES DS:0 DS:1 DS:2 TOTAL
22932024-02-27T21:44:52.674Z New 0 0 0 0
22942024-02-27T21:44:52.674Z Sent 27 1 0 28
22952024-02-27T21:44:52.674Z Done 3 29 30 62
22962024-02-27T21:44:52.675Z Skipped 0 0 0 0
22972024-02-27T21:44:52.675Z Error 0 0 0 0
22982024-02-27T21:44:52.675ZLast Flush: 0 1024 1024
22992024-02-27T21:44:52.675ZDownstairs last five completed:
23002024-02-27T21:44:52.675ZUpstairs last five completed: 30 29 28 27 26
23012024-02-27T21:44:52.675Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:50.438625553Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1218}
23022024-02-27T21:44:52.676ZWrote out file "/tmp/test_repair_verify.data"
23032024-02-27T21:44:52.676Z
23042024-02-27T21:44:52.676ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23052024-02-27T21:44:52.676ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23062024-02-27T21:44:52.677Z 0 000-019 10 10 10 18 18 18 F F F
23072024-02-27T21:44:52.677Z 1 020-039 16 16 16 34 34 34 F F F
23082024-02-27T21:44:52.677Z 2 040-059 18 18 18 35 35 35 F F F
23092024-02-27T21:44:52.677Z 3 060-079 19 19 19 39 39 39 F F F
23102024-02-27T21:44:52.677Z 4 080-099 18 18 18 37 37 37 F F F
23112024-02-27T21:44:52.677Z 5 100-119 8 8 8 16 16 16 F F F
23122024-02-27T21:44:52.678Z 6 120-139 16 16 16 34 34 34 F F F
23132024-02-27T21:44:52.678Z 7 140-159 11 20 20 21 41 41 F F F <---
23142024-02-27T21:44:52.678Z 8 160-179 12 12 12 27 27 27 F F F
23152024-02-27T21:44:52.678Z 9 180-199 18 18 18 35 35 35 F F F
23162024-02-27T21:44:52.678Z 10 200-219 14 14 14 30 30 30 F F F
23172024-02-27T21:44:52.678Z 11 220-239 19 20 20 39 42 42 F F F <---
23182024-02-27T21:44:52.678Z 12 240-259 16 16 16 34 34 34 F F F
23192024-02-27T21:44:52.679Z 13 260-279 18 18 18 35 35 35 F F F
23202024-02-27T21:44:52.679Z 14 280-299 16 20 20 34 40 40 F F F <---
23212024-02-27T21:44:52.679Z 15 300-319 18 20 20 38 40 40 F F F <---
23222024-02-27T21:44:52.679Z 16 320-339 16 20 20 34 40 40 F F F <---
23232024-02-27T21:44:52.679Z 17 340-359 19 19 19 39 39 39 F F F
23242024-02-27T21:44:52.680Z 18 360-379 16 16 16 34 34 34 F F F
23252024-02-27T21:44:52.680Z 19 380-399 16 16 16 34 34 34 F F F
23262024-02-27T21:44:52.680Z 20 400-419 19 20 20 39 40 40 F T T <---
23272024-02-27T21:44:52.680Z 21 420-439 19 19 19 39 39 39 F F F
23282024-02-27T21:44:52.680Z 22 440-459 12 20 20 26 40 40 F F F <---
23292024-02-27T21:44:52.680Z 23 460-479 8 20 20 16 40 40 T F F <---
23302024-02-27T21:44:52.681Z 24 480-499 19 19 19 39 39 39 F T T <---
23312024-02-27T21:44:52.681Z 25 500-519 19 20 20 39 43 43 F F F <---
23322024-02-27T21:44:52.681Z 26 520-539 11 11 11 21 21 21 F F F
23332024-02-27T21:44:52.681Z 27 540-559 18 20 20 36 40 40 F F F <---
23342024-02-27T21:44:52.682Z 28 560-579 18 20 20 36 40 40 F F F <---
23352024-02-27T21:44:52.682Z 29 580-599 13 13 13 28 28 28 F F F
23362024-02-27T21:44:52.682ZMax gen: 20, Max flush: 43
23372024-02-27T21:44:52.682ZError: Difference in extent metadata found!
23382024-02-27T21:44:52.682ZA Difference in extent metadata is expected here
23392024-02-27T21:44:52.683ZOn loop 10
23402024-02-27T21:44:52.683Z
23412024-02-27T21:44:52.683ZVerifying data now
23422024-02-27T21:44:56.680ZNo change after verify
23432024-02-27T21:44:56.681ZLoop: 10 Downstairs dump after verify (and repair):
23442024-02-27T21:44:56.702ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23452024-02-27T21:44:56.702Z 0 000-019 10 10 10 18 18 18 F F F
23462024-02-27T21:44:56.702Z 1 020-039 16 16 16 34 34 34 F F F
23472024-02-27T21:44:56.702Z 2 040-059 18 18 18 35 35 35 F F F
23482024-02-27T21:44:56.703Z 3 060-079 19 19 19 39 39 39 F F F
23492024-02-27T21:44:56.703Z 4 080-099 18 18 18 37 37 37 F F F
23502024-02-27T21:44:56.703Z 5 100-119 8 8 8 16 16 16 F F F
23512024-02-27T21:44:56.703Z 6 120-139 16 16 16 34 34 34 F F F
23522024-02-27T21:44:56.703Z 7 140-159 20 20 20 41 41 41 F F F
23532024-02-27T21:44:56.704Z 8 160-179 12 12 12 27 27 27 F F F
23542024-02-27T21:44:56.704Z 9 180-199 18 18 18 35 35 35 F F F
23552024-02-27T21:44:56.704Z 10 200-219 14 14 14 30 30 30 F F F
23562024-02-27T21:44:56.704Z 11 220-239 20 20 20 42 42 42 F F F
23572024-02-27T21:44:56.704Z 12 240-259 16 16 16 34 34 34 F F F
23582024-02-27T21:44:56.704Z 13 260-279 18 18 18 35 35 35 F F F
23592024-02-27T21:44:56.704Z 14 280-299 20 20 20 40 40 40 F F F
23602024-02-27T21:44:56.705Z 15 300-319 20 20 20 40 40 40 F F F
23612024-02-27T21:44:56.705Z 16 320-339 20 20 20 40 40 40 F F F
23622024-02-27T21:44:56.705Z 17 340-359 19 19 19 39 39 39 F F F
23632024-02-27T21:44:56.705Z 18 360-379 16 16 16 34 34 34 F F F
23642024-02-27T21:44:56.706Z 19 380-399 16 16 16 34 34 34 F F F
23652024-02-27T21:44:56.706Z 20 400-419 21 21 21 44 44 44 F F F
23662024-02-27T21:44:56.706Z 21 420-439 19 19 19 39 39 39 F F F
23672024-02-27T21:44:56.706Z 22 440-459 20 20 20 40 40 40 F F F
23682024-02-27T21:44:56.707Z 23 460-479 20 20 20 40 40 40 F F F
23692024-02-27T21:44:56.707Z 24 480-499 21 21 21 44 44 44 F F F
23702024-02-27T21:44:56.707Z 25 500-519 20 20 20 43 43 43 F F F
23712024-02-27T21:44:56.707Z 26 520-539 11 11 11 21 21 21 F F F
23722024-02-27T21:44:56.707Z 27 540-559 20 20 20 40 40 40 F F F
23732024-02-27T21:44:56.708Z 28 560-579 20 20 20 40 40 40 F F F
23742024-02-27T21:44:56.708Z 29 580-599 13 13 13 28 28 28 F F F
23752024-02-27T21:44:56.708ZMax gen: 21, Max flush: 44
23762024-02-27T21:44:56.708Z
23772024-02-27T21:44:56.708ZBegin loop 11
23782024-02-27T21:44:56.709ZDownstairs to restart: 2
23792024-02-27T21:44:56.764Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.11798407Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226}
23802024-02-27T21:44:56.764Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\nCrucible runtime is spawned
23812024-02-27T21:44:56.764Z git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118357416Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226}
23822024-02-27T21:44:56.764Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118416699Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226}
23832024-02-27T21:44:56.764Z{"msg":"Crucible 99bbbe41-a59b-437f-8bbf-0f366852e2d9 has session id: bb4b501a-ecad-46ef-85d4-d539c28b6b68","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118447725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23842024-02-27T21:44:56.764Z{"msg":"Upstairs opts: Upstairs UUID: 99bbbe41-a59b-437f-8bbf-0f366852e2d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118503838Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23852024-02-27T21:44:56.764Z{"msg":"Crucible stats registered with UUID: 99bbbe41-a59b-437f-8bbf-0f366852e2d9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118548013Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23862024-02-27T21:44:56.764Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118592088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226}
23872024-02-27T21:44:56.764Z{"msg":"99bbbe41-a59b-437f-8bbf-0f366852e2d9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118632463Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23882024-02-27T21:44:56.765Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118673518Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23892024-02-27T21:44:56.765Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118712894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23902024-02-27T21:44:56.765Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118737951Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23912024-02-27T21:44:56.765Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118773907Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23922024-02-27T21:44:56.765Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118804753Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23932024-02-27T21:44:56.765Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.11883137Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23942024-02-27T21:44:56.765Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118857277Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23952024-02-27T21:44:56.765Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.118888903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23962024-02-27T21:44:56.766Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.11891513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23972024-02-27T21:44:56.766Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.119140443Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"io task","client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23982024-02-27T21:44:56.766Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.119227663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"io task","client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
23992024-02-27T21:44:56.766Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.119267049Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"io task","client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24002024-02-27T21:44:56.766Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.119924011Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24012024-02-27T21:44:56.766Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.119966556Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24022024-02-27T21:44:56.766Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.119992823Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24032024-02-27T21:44:56.766Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.120058135Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24042024-02-27T21:44:56.766Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.12010222Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24052024-02-27T21:44:56.766Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.120137786Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24062024-02-27T21:44:56.766Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.120622309Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24072024-02-27T21:44:56.766Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.120664394Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24082024-02-27T21:44:56.767Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.12078296Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24092024-02-27T21:44:56.767Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.12129287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24102024-02-27T21:44:56.767Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121335375Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24112024-02-27T21:44:56.767Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121351023Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24122024-02-27T21:44:56.767Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121371581Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24132024-02-27T21:44:56.767Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121385679Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24142024-02-27T21:44:56.767Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121406017Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24152024-02-27T21:44:56.767Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121419705Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24162024-02-27T21:44:56.768Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121440783Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24172024-02-27T21:44:56.768Z{"msg":"[0]R flush_numbers[0..12]: [18, 34, 35, 39, 37, 16, 34, 41, 27, 35, 30, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121455081Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24182024-02-27T21:44:56.768Z{"msg":"[0]R generation[0..12]: [10, 16, 18, 19, 18, 8, 16, 20, 12, 18, 14, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121468829Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24192024-02-27T21:44:56.768Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121483398Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24202024-02-27T21:44:56.768Z{"msg":"[1]R flush_numbers[0..12]: [18, 34, 35, 39, 37, 16, 34, 41, 27, 35, 30, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121497326Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24212024-02-27T21:44:56.768Z{"msg":"[1]R generation[0..12]: [10, 16, 18, 19, 18, 8, 16, 20, 12, 18, 14, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.12155095Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24222024-02-27T21:44:56.768Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121565858Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
24232024-02-27T21:44:56.768Zbb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24242024-02-27T21:44:56.769Z{"msg":"[2]R flush_numbers[0..12]: [18, 34, 35, 39, 37, 16, 34, 41, 27, 35, 30, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121594574Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24252024-02-27T21:44:56.769Z{"msg":"[2]R generation[0..12]: [10, 16, 18, 19, 18, 8, 16, 20, 12, 18, 14, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121615612Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24262024-02-27T21:44:56.769Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.12163012Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24272024-02-27T21:44:56.769Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121660407Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24282024-02-27T21:44:56.769Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121674925Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24292024-02-27T21:44:56.769Z{"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121695702Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24302024-02-27T21:44:56.769Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121709361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"":"downstairs","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24312024-02-27T21:44:56.769Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121729249Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24322024-02-27T21:44:56.769Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121741477Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"0","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24332024-02-27T21:44:56.769Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121761395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"1","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24342024-02-27T21:44:56.769Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121774833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"client":"2","":"downstairs","session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24352024-02-27T21:44:56.770Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121794611Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24362024-02-27T21:44:56.770Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121807409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24372024-02-27T21:44:56.770Z{"msg":"99bbbe41-a59b-437f-8bbf-0f366852e2d9 is now active with session: bb4b501a-ecad-46ef-85d4-d539c28b6b68","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121826977Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24382024-02-27T21:44:56.770Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121839216Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226,"session_id":"bb4b501a-ecad-46ef-85d4-d539c28b6b68"}
24392024-02-27T21:44:56.770Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:55.121857963Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226}
24402024-02-27T21:44:56.770ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24412024-02-27T21:44:56.770ZLoading write count information from file "/tmp/test_repair_verify.data"
24422024-02-27T21:44:56.771ZRun Repair workload
24432024-02-27T21:44:56.771Z01/30 Write block 478 len 24576 data: 2 2 2 2 2 2
24442024-02-27T21:44:56.771Z02/30 Read block 275 len 20480
24452024-02-27T21:44:56.771Z03/30 Write block 450 len 28672 data: 4 4 3 3 2 2 2
24462024-02-27T21:44:56.771Z04/30 Read block 27 len 8192
24472024-02-27T21:44:56.771Z05/30 Write block 111 len 20480 data: 3 3 2 2 2
24482024-02-27T21:44:56.771Z06/30 Write block 228 len 36864 data: 6 6 7 8 8 7 6 6 4
24492024-02-27T21:44:56.771Z07/30 Write block 393 len 40960 data: 4 2 2 2 2 2 2 3 4 3
24502024-02-27T21:44:56.771Z08/30 Write block 184 len 40960 data: 2 2 2 2 3 4 4 4 3 3
24512024-02-27T21:44:56.771Z09/30 Read block 564 len 4096
24522024-02-27T21:44:56.772Z10/30 Read block 341 len 40960
24532024-02-27T21:44:56.772Z11/30 Read block 496 len 28672
24542024-02-27T21:44:56.772Z12/30 Read block 74 len 4096
24552024-02-27T21:44:56.772Z13/30 Write block 336 len 8192 data: 5 6
24562024-02-27T21:44:56.772Z14/30 Flush
24572024-02-27T21:44:56.802Z15/30 Write block 481 len 20480 data: 3 3 3 2 2
24582024-02-27T21:44:56.803Z16/30 Write block 316 len 24576 data: 4 2 2 2 2 2
24592024-02-27T21:44:56.803Z17/30 Write block 527 len 16384 data: 2 2 2 2
24602024-02-27T21:44:56.803Z18/30 Read block 67 len 12288
24612024-02-27T21:44:56.803Z19/30 Flush
24622024-02-27T21:44:56.803Z20/30 Write block 459 len 40960 data: 3 2 3 4 4 3 3 3 2 2
24632024-02-27T21:44:56.803Z21/30 Read block 132 len 8192
24642024-02-27T21:44:56.803Z22/30 Read block 247 len 28672
24652024-02-27T21:44:56.803Z23/30 Read block 476 len 8192
24662024-02-27T21:44:56.804Z24/30 Write block 244 len 40960 data: 2 3 3 4 3 3 3 3 3 3
24672024-02-27T21:44:56.804Z25/30 Write block 159 len 8192 data: 4 3
24682024-02-27T21:44:56.804Z26/30 Read block 203 len 12288
24692024-02-27T21:44:56.804Z27/30 Read block 136 len 32768
24702024-02-27T21:44:56.804Z28/30 Read block 67 len 4096
24712024-02-27T21:44:56.804Z29/30 Read block 248 len 40960
24722024-02-27T21:44:56.804Z30/30 Write block 489 len 20480 data: 4 4 5 6 5
24732024-02-27T21:44:56.804Z----------------------------------------------------------------
24742024-02-27T21:44:56.805Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
24752024-02-27T21:44:56.805ZStates: Active Active Active
24762024-02-27T21:44:56.805ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24772024-02-27T21:44:56.805Z 1 Acked 1000 Write 6 Done Done Done false
24782024-02-27T21:44:56.805Z 2 Acked 1001 Read 5 Done Done Done false
24792024-02-27T21:44:56.805Z 3 Acked 1002 Write 7 Done Done Sent false
24802024-02-27T21:44:56.805Z 4 Acked 1003 Read 2 Done Done Sent false
24812024-02-27T21:44:56.805Z 5 Acked 1004 Write 5 Done Done Sent false
24822024-02-27T21:44:56.806Z 6 Acked 1005 Write 9 Done Done Sent false
24832024-02-27T21:44:56.806Z 7 Acked 1006 Write 10 Done Done Sent false
24842024-02-27T21:44:56.806Z 8 Acked 1007 Write 10 Done Done Sent false
24852024-02-27T21:44:56.806Z 9 Acked 1008 Read 1 Done Done Sent false
24862024-02-27T21:44:56.806Z 10 Acked 1009 Read 10 Done Done Sent false
24872024-02-27T21:44:56.806Z 11 Acked 1010 Read 7 Done Done Sent false
24882024-02-27T21:44:56.807Z 12 Acked 1011 Read 1 Done Done Sent false
24892024-02-27T21:44:56.807Z 13 Acked 1012 Write 2 Done Done Sent false
24902024-02-27T21:44:56.807Z 14 Acked 1013 Flush 0 Done Done Sent false
24912024-02-27T21:44:56.807Z 15 Acked 1014 Write 5 Done Done Sent false
24922024-02-27T21:44:56.807Z 16 Acked 1015 Write 6 Done Done Sent false
24932024-02-27T21:44:56.807Z 17 Acked 1016 Write 4 Done Done Sent false
24942024-02-27T21:44:56.807Z 18 Acked 1017 Read 3 Done Done Sent false
24952024-02-27T21:44:56.807Z 19 Acked 1018 Flush 0 Done Done Sent false
24962024-02-27T21:44:56.808Z 20 Acked 1019 Write 10 Done Done Sent false
24972024-02-27T21:44:56.808Z 21 Acked 1020 Read 2 Done Done Sent false
24982024-02-27T21:44:56.808Z 22 Acked 1021 Read 7 Done Done Sent false
24992024-02-27T21:44:56.808Z 23 Acked 1022 Read 2 Done Done Sent false
25002024-02-27T21:44:56.808Z 24 Acked 1023 Write 10 Done Done Sent false
25012024-02-27T21:44:56.808Z 25 Acked 1024 Write 2 Done Done Sent false
25022024-02-27T21:44:56.808Z 26 Acked 1025 Read 3 Done Done Sent false
25032024-02-27T21:44:56.808Z 27 Acked 1026 Read 8 Done Done Sent false
25042024-02-27T21:44:56.808Z 28 Acked 1027 Read 1 Done Done Sent false
25052024-02-27T21:44:56.809Z 29 Acked 1028 Read 10 Done Done Sent false
25062024-02-27T21:44:56.809Z 30 Acked 1029 Write 5 Sent Sent Sent false
25072024-02-27T21:44:56.809Z STATES DS:0 DS:1 DS:2 TOTAL
25082024-02-27T21:44:56.809Z New 0 0 0 0
25092024-02-27T21:44:56.809Z Sent 1 1 28 30
25102024-02-27T21:44:56.809Z Done 29 29 2 60
25112024-02-27T21:44:56.809Z Skipped 0 0 0 0
25122024-02-27T21:44:56.809Z Error 0 0 0 0
25132024-02-27T21:44:56.809ZLast Flush: 1018 1018 0
25142024-02-27T21:44:56.809ZDownstairs last five completed:
25152024-02-27T21:44:56.809ZUpstairs last five completed: 30 29 28 27 26
25162024-02-27T21:44:56.809Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:55.15244622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1226}
25172024-02-27T21:44:56.809ZWrote out file "/tmp/test_repair_verify.data"
25182024-02-27T21:44:56.809Z
25192024-02-27T21:44:56.810ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25202024-02-27T21:44:56.810ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25212024-02-27T21:44:56.810Z 0 000-019 10 10 10 18 18 18 F F F
25222024-02-27T21:44:56.810Z 1 020-039 16 16 16 34 34 34 F F F
25232024-02-27T21:44:56.810Z 2 040-059 18 18 18 35 35 35 F F F
25242024-02-27T21:44:56.810Z 3 060-079 19 19 19 39 39 39 F F F
25252024-02-27T21:44:56.810Z 4 080-099 18 18 18 37 37 37 F F F
25262024-02-27T21:44:56.810Z 5 100-119 22 22 8 45 45 16 F F F <---
25272024-02-27T21:44:56.810Z 6 120-139 16 16 16 34 34 34 F F F
25282024-02-27T21:44:56.810Z 7 140-159 20 20 20 41 41 41 T T F <---
25292024-02-27T21:44:56.811Z 8 160-179 12 12 12 27 27 27 T T F <---
25302024-02-27T21:44:56.811Z 9 180-199 22 22 18 45 45 35 F F F <---
25312024-02-27T21:44:56.811Z 10 200-219 14 14 14 30 30 30 F F F
25322024-02-27T21:44:56.811Z 11 220-239 22 22 20 45 45 42 F F F <---
25332024-02-27T21:44:56.811Z 12 240-259 16 16 16 34 34 34 T T F <---
25342024-02-27T21:44:56.811Z 13 260-279 18 18 18 35 35 35 F F F
25352024-02-27T21:44:56.811Z 14 280-299 20 20 20 40 40 40 F F F
25362024-02-27T21:44:56.811Z 15 300-319 22 22 20 46 46 40 F F F <---
25372024-02-27T21:44:56.811Z 16 320-339 22 22 20 46 46 40 F F F <---
25382024-02-27T21:44:56.811Z 17 340-359 19 19 19 39 39 39 F F F
25392024-02-27T21:44:56.811Z 18 360-379 16 16 16 34 34 34 F F F
25402024-02-27T21:44:56.811Z 19 380-399 22 22 16 45 45 34 F F F <---
25412024-02-27T21:44:56.812Z 20 400-419 22 22 21 45 45 44 F F F <---
25422024-02-27T21:44:56.812Z 21 420-439 19 19 19 39 39 39 F F F
25432024-02-27T21:44:56.812Z 22 440-459 22 22 20 45 45 40 T T F <---
25442024-02-27T21:44:56.812Z 23 460-479 22 22 20 45 45 40 T T T <---
25452024-02-27T21:44:56.812Z 24 480-499 22 22 21 46 46 44 T T T <---
25462024-02-27T21:44:56.812Z 25 500-519 20 20 20 43 43 43 F F F
25472024-02-27T21:44:56.812Z 26 520-539 22 22 11 46 46 21 F F F <---
25482024-02-27T21:44:56.812Z 27 540-559 20 20 20 40 40 40 F F F
25492024-02-27T21:44:56.812Z 28 560-579 20 20 20 40 40 40 F F F
25502024-02-27T21:44:56.812Z 29 580-599 13 13 13 28 28 28 F F F
25512024-02-27T21:44:56.812ZMax gen: 22, Max flush: 46
25522024-02-27T21:44:56.813ZError: Difference in extent metadata found!
25532024-02-27T21:44:56.813ZA Difference in extent metadata is expected here
25542024-02-27T21:44:56.813ZOn loop 11
25552024-02-27T21:44:56.813Z
25562024-02-27T21:44:56.813ZVerifying data now
25572024-02-27T21:45:01.467ZNo change after verify
25582024-02-27T21:45:01.467ZLoop: 11 Downstairs dump after verify (and repair):
25592024-02-27T21:45:01.490ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25602024-02-27T21:45:01.490Z 0 000-019 10 10 10 18 18 18 F F F
25612024-02-27T21:45:01.490Z 1 020-039 16 16 16 34 34 34 F F F
25622024-02-27T21:45:01.490Z 2 040-059 18 18 18 35 35 35 F F F
25632024-02-27T21:45:01.490Z 3 060-079 19 19 19 39 39 39 F F F
25642024-02-27T21:45:01.490Z 4 080-099 18 18 18 37 37 37 F F F
25652024-02-27T21:45:01.490Z 5 100-119 22 22 22 45 45 45 F F F
25662024-02-27T21:45:01.490Z 6 120-139 16 16 16 34 34 34 F F F
25672024-02-27T21:45:01.490Z 7 140-159 23 23 23 47 47 47 F F F
25682024-02-27T21:45:01.490Z 8 160-179 23 23 23 47 47 47 F F F
25692024-02-27T21:45:01.490Z 9 180-199 22 22 22 45 45 45 F F F
25702024-02-27T21:45:01.490Z 10 200-219 14 14 14 30 30 30 F F F
25712024-02-27T21:45:01.490Z 11 220-239 22 22 22 45 45 45 F F F
25722024-02-27T21:45:01.490Z 12 240-259 23 23 23 47 47 47 F F F
25732024-02-27T21:45:01.490Z 13 260-279 18 18 18 35 35 35 F F F
25742024-02-27T21:45:01.490Z 14 280-299 20 20 20 40 40 40 F F F
25752024-02-27T21:45:01.490Z 15 300-319 22 22 22 46 46 46 F F F
25762024-02-27T21:45:01.491Z 16 320-339 22 22 22 46 46 46 F F F
25772024-02-27T21:45:01.491Z 17 340-359 19 19 19 39 39 39 F F F
25782024-02-27T21:45:01.491Z 18 360-379 16 16 16 34 34 34 F F F
25792024-02-27T21:45:01.491Z 19 380-399 22 22 22 45 45 45 F F F
25802024-02-27T21:45:01.491Z 20 400-419 22 22 22 45 45 45 F F F
25812024-02-27T21:45:01.491Z 21 420-439 19 19 19 39 39 39 F F F
25822024-02-27T21:45:01.491Z 22 440-459 23 23 23 47 47 47 F F F
25832024-02-27T21:45:01.491Z 23 460-479 23 23 23 47 47 47 F F F
25842024-02-27T21:45:01.491Z 24 480-499 23 23 23 47 47 47 F F F
25852024-02-27T21:45:01.491Z 25 500-519 20 20 20 43 43 43 F F F
25862024-02-27T21:45:01.491Z 26 520-539 22 22 22 46 46 46 F F F
25872024-02-27T21:45:01.492Z 27 540-559 20 20 20 40 40 40 F F F
25882024-02-27T21:45:01.492Z 28 560-579 20 20 20 40 40 40 F F F
25892024-02-27T21:45:01.492Z 29 580-599 13 13 13 28 28 28 F F F
25902024-02-27T21:45:01.492ZMax gen: 23, Max flush: 47
25912024-02-27T21:45:01.492Z
25922024-02-27T21:45:01.492ZBegin loop 12
25932024-02-27T21:45:01.492ZDownstairs to restart: 2
25942024-02-27T21:45:03.362Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897289325Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234}
25952024-02-27T21:45:03.362Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
25962024-02-27T21:45:03.363Z,"time":"2024-02-27T21:44:59.897608507Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234}
25972024-02-27T21:45:03.363Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897658341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234}
25982024-02-27T21:45:03.363Z{"msg":"Crucible 96c00913-7dc6-46e2-9c1a-3c450dfc9333 has session id: 9346d755-7e30-4570-81dd-8490eaab0b04","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897705606Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
25992024-02-27T21:45:03.363Z{"msg":"Upstairs opts: Upstairs UUID: 96c00913-7dc6-46e2-9c1a-3c450dfc9333, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897745151Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26002024-02-27T21:45:03.363Z{"msg":"Crucible stats registered with UUID: 96c00913-7dc6-46e2-9c1a-3c450dfc9333","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897762999Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26012024-02-27T21:45:03.363Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897779687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234}
26022024-02-27T21:45:03.363Z{"msg":"96c00913-7dc6-46e2-9c1a-3c450dfc9333 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897867397Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26032024-02-27T21:45:03.363Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897890024Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26042024-02-27T21:45:03.363Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897933509Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26052024-02-27T21:45:03.364Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.897977074Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26062024-02-27T21:45:03.364Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.898025368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26072024-02-27T21:45:03.364Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.898057654Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26082024-02-27T21:45:03.364Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.898083831Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26092024-02-27T21:45:03.364Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.898124996Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26102024-02-27T21:45:03.364Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.898241773Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"io task","client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26112024-02-27T21:45:03.364Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.898295776Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"io task","client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26122024-02-27T21:45:03.364Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.898323803Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"io task","client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26132024-02-27T21:45:03.364Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.8983482Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"io task","client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26142024-02-27T21:45:03.364Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.898370467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"io task","client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26152024-02-27T21:45:03.364Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.89894646Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26162024-02-27T21:45:03.364Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.898987765Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26172024-02-27T21:45:03.365Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.899003743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26182024-02-27T21:45:03.365Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.89903141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26192024-02-27T21:45:03.365Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.899051547Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26202024-02-27T21:45:03.365Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.899082794Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26212024-02-27T21:45:03.365Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.899629429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26222024-02-27T21:45:03.365Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.899732217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26232024-02-27T21:45:03.365Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.899784591Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26242024-02-27T21:45:03.365Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900056159Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26252024-02-27T21:45:03.365Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900090205Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26262024-02-27T21:45:03.365Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900115232Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26272024-02-27T21:45:03.366Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900264124Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26282024-02-27T21:45:03.366Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.90030142Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26292024-02-27T21:45:03.366Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900317558Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26302024-02-27T21:45:03.366Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900392489Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26312024-02-27T21:45:03.366Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900412007Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26322024-02-27T21:45:03.366Z{"msg":"[0]R flush_numbers[0..12]: [18, 34, 35, 39, 37, 45, 34, 47, 47, 45, 30, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900434944Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26332024-02-27T21:45:03.366Z{"msg":"[0]R generation[0..12]: [10, 16, 18, 19, 18, 22, 16, 23, 23, 22, 14, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900449402Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26342024-02-27T21:45:03.367Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900463851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26352024-02-27T21:45:03.367Z{"msg":"[1]R flush_numbers[0..12]: [18, 34, 35, 39, 37, 45, 34, 47, 47, 45, 30, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900484689Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26362024-02-27T21:45:03.367Z{"msg":"[1]R generation[0..12]: [10, 16, 18, 19, 18, 22, 16, 23, 23, 22, 14, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900498637Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26372024-02-27T21:45:03.367Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900512765Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26382024-02-27T21:45:03.367Z{"msg":"[2]R flush_numbers[0..12]: [18, 34, 35, 39, 37, 45, 34, 47, 47, 45, 30, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900533893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26392024-02-27T21:45:03.367ZWait for a query_work_queue command to finish before sending IO
26402024-02-27T21:45:03.367Z{"msg":"[2]R generation[0..12]: [10, 16, 18, 19, 18, 22, 16, 23, 23, 22, 14, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.90055251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26412024-02-27T21:45:03.367Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900573988Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26422024-02-27T21:45:03.367Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900588236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26432024-02-27T21:45:03.368Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900608774Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26442024-02-27T21:45:03.368Z{"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900622332Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26452024-02-27T21:45:03.368Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900647249Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"":"downstairs","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26462024-02-27T21:45:03.368Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900661968Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26472024-02-27T21:45:03.368Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900674156Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"0","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26482024-02-27T21:45:03.368Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900694574Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"1","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26492024-02-27T21:45:03.368Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900708112Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"client":"2","":"downstairs","session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26502024-02-27T21:45:03.368Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.90072215Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26512024-02-27T21:45:03.368Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900734569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26522024-02-27T21:45:03.369Z{"msg":"96c00913-7dc6-46e2-9c1a-3c450dfc9333 is now active with session: 9346d755-7e30-4570-81dd-8490eaab0b04","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900747618Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26532024-02-27T21:45:03.369Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900767205Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234,"session_id":"9346d755-7e30-4570-81dd-8490eaab0b04"}
26542024-02-27T21:45:03.369Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:44:59.900779834Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234}
26552024-02-27T21:45:03.369ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26562024-02-27T21:45:03.369ZLoading write count information from file "/tmp/test_repair_verify.data"
26572024-02-27T21:45:03.369ZRun Repair workload
26582024-02-27T21:45:03.369Z01/30 Write block 228 len 40960 data: 7 7 8 9 9 8 7 7 5 3
26592024-02-27T21:45:03.369Z02/30 Read block 33 len 4096
26602024-02-27T21:45:03.369Z03/30 Write block 434 len 4096 data: 4
26612024-02-27T21:45:03.369Z04/30 Read block 342 len 32768
26622024-02-27T21:45:03.370Z05/30 Flush
26632024-02-27T21:45:03.370Z06/30 Write block 188 len 28672 data: 4 5 5 5 4 4 3
26642024-02-27T21:45:03.370Z07/30 Read block 185 len 16384
26652024-02-27T21:45:03.370Z08/30 Write block 58 len 36864 data: 4 3 3 3 3 3 2 2 2
26662024-02-27T21:45:03.370Z09/30 Write block 480 len 24576 data: 3 4 4 4 3 3
26672024-02-27T21:45:03.370Z10/30 Read block 137 len 24576
26682024-02-27T21:45:03.370Z11/30 Flush
26692024-02-27T21:45:03.370Z12/30 Write block 448 len 36864 data: 5 4 5 5 4 4 3 3 3
26702024-02-27T21:45:03.370Z13/30 Read block 393 len 36864
26712024-02-27T21:45:03.370Z14/30 Read block 130 len 24576
26722024-02-27T21:45:03.370Z15/30 Write block 84 len 8192 data: 3 3
26732024-02-27T21:45:03.370Z16/30 Read block 429 len 16384
26742024-02-27T21:45:03.370Z17/30 Write block 53 len 32768 data: 3 3 2 4 4 5 4 4
26752024-02-27T21:45:03.370Z18/30 Write block 325 len 24576 data: 2 2 2 3 3 3
26762024-02-27T21:45:03.370Z19/30 Read block 128 len 36864
26772024-02-27T21:45:03.371Z20/30 Read block 55 len 36864
26782024-02-27T21:45:03.371Z21/30 Read block 56 len 8192
26792024-02-27T21:45:03.371Z22/30 Write block 186 len 28672 data: 3 3 5 6 6 6 5
26802024-02-27T21:45:03.371Z23/30 Write block 146 len 36864 data: 3 3 3 3 3 3 3 4 3
26812024-02-27T21:45:03.371Z24/30 Read block 490 len 20480
26822024-02-27T21:45:03.371Z25/30 Write block 154 len 40960 data: 4 5 6 6 5 5 4 4 3 3
26832024-02-27T21:45:03.371Z26/30 Write block 29 len 24576 data: 3 4 3 2 2 2
26842024-02-27T21:45:03.371Z27/30 Write block 34 len 28672 data: 3 2 2 2 3 4 4
26852024-02-27T21:45:03.371Z28/30 Write block 437 len 16384 data: 3 2 2 2
26862024-02-27T21:45:03.371Z29/30 Read block 150 len 36864
26872024-02-27T21:45:03.371Z30/30 Write block 261 len 4096 data: 3
26882024-02-27T21:45:03.371Z----------------------------------------------------------------
26892024-02-27T21:45:03.371Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:25
26902024-02-27T21:45:03.372ZStates: Active Active Active
26912024-02-27T21:45:03.372ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26922024-02-27T21:45:03.372Z 6 Acked 1005 Write 7 Done Done Done false
26932024-02-27T21:45:03.372Z 7 Acked 1006 Read 4 Done Done Done false
26942024-02-27T21:45:03.398Z 8 Acked 1007 Write 9 Done Done Done false
26952024-02-27T21:45:03.398Z 9 Acked 1008 Write 6 Done Done Done false
26962024-02-27T21:45:03.398Z 10 Acked 1009 Read 6 Done Done Done false
26972024-02-27T21:45:03.398Z 11 Acked 1010 Flush 0 Done Done Sent false
26982024-02-27T21:45:03.399Z 12 Acked 1011 Write 9 Done Done Sent false
26992024-02-27T21:45:03.399Z 13 Acked 1012 Read 9 Done Done Sent false
27002024-02-27T21:45:03.399Z 14 Acked 1013 Read 6 Done Done Sent false
27012024-02-27T21:45:03.399Z 15 Acked 1014 Write 2 Done Done Sent false
27022024-02-27T21:45:03.399Z 16 Acked 1015 Read 4 Done Done Sent false
27032024-02-27T21:45:03.399Z 17 Acked 1016 Write 8 Done Done Sent false
27042024-02-27T21:45:03.399Z 18 Acked 1017 Write 6 Done Done Sent false
27052024-02-27T21:45:03.399Z 19 Acked 1018 Read 9 Done Done Sent false
27062024-02-27T21:45:03.400Z 20 Acked 1019 Read 9 Done Done Sent false
27072024-02-27T21:45:03.400Z 21 Acked 1020 Read 2 Done Done Sent false
27082024-02-27T21:45:03.400Z 22 Acked 1021 Write 7 Done Done Sent false
27092024-02-27T21:45:03.400Z 23 Acked 1022 Write 9 Done Done Sent false
27102024-02-27T21:45:03.400Z 24 Acked 1023 Read 5 Done Done Sent false
27112024-02-27T21:45:03.400Z 25 Acked 1024 Write 10 Done Done Sent false
27122024-02-27T21:45:03.400Z 26 Acked 1025 Write 6 Done Done Sent false
27132024-02-27T21:45:03.400Z 27 Acked 1026 Write 7 Done Done Sent false
27142024-02-27T21:45:03.400Z 28 Acked 1027 Write 4 Done Done Sent false
27152024-02-27T21:45:03.400Z 29 Acked 1028 Read 9 Done Sent Sent false
27162024-02-27T21:45:03.400Z 30 Acked 1029 Write 1 Sent Sent Sent false
27172024-02-27T21:45:03.401Z STATES DS:0 DS:1 DS:2 TOTAL
27182024-02-27T21:45:03.401Z New 0 0 0 0
27192024-02-27T21:45:03.401Z Sent 1 2 20 23
27202024-02-27T21:45:03.401Z Done 24 23 5 52
27212024-02-27T21:45:03.401Z Skipped 0 0 0 0
27222024-02-27T21:45:03.401Z Error 0 0 0 0
27232024-02-27T21:45:03.401ZLast Flush: 1010 1010 1004
27242024-02-27T21:45:03.401ZDownstairs last five completed: 1004 1003 1002 1001 1000
27252024-02-27T21:45:03.401ZUpstairs last five completed: 30 29 28 27 26
27262024-02-27T21:45:03.402Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:44:59.93264621Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1234}
27272024-02-27T21:45:03.402ZWrote out file "/tmp/test_repair_verify.data"
27282024-02-27T21:45:03.402Z
27292024-02-27T21:45:03.402ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27302024-02-27T21:45:03.402ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27312024-02-27T21:45:03.402Z 0 000-019 10 10 10 18 18 18 F F F
27322024-02-27T21:45:03.402Z 1 020-039 16 16 16 34 34 34 T T F <---
27332024-02-27T21:45:03.402Z 2 040-059 24 24 18 49 49 35 T T T <---
27342024-02-27T21:45:03.402Z 3 060-079 24 24 19 49 49 39 T T T <---
27352024-02-27T21:45:03.403Z 4 080-099 18 18 18 37 37 37 T T F <---
27362024-02-27T21:45:03.403Z 5 100-119 22 22 22 45 45 45 F F F
27372024-02-27T21:45:03.403Z 6 120-139 16 16 16 34 34 34 F F F
27382024-02-27T21:45:03.403Z 7 140-159 23 23 23 47 47 47 T T F <---
27392024-02-27T21:45:03.403Z 8 160-179 23 23 23 47 47 47 T T F <---
27402024-02-27T21:45:03.403Z 9 180-199 24 24 22 49 49 45 T T T <---
27412024-02-27T21:45:03.403Z 10 200-219 14 14 14 30 30 30 F F F
27422024-02-27T21:45:03.403Z 11 220-239 24 24 24 48 48 48 F F F
27432024-02-27T21:45:03.404Z 12 240-259 23 23 23 47 47 47 F F F
27442024-02-27T21:45:03.404Z 13 260-279 18 18 18 35 35 35 T T F <---
27452024-02-27T21:45:03.404Z 14 280-299 20 20 20 40 40 40 F F F
27462024-02-27T21:45:03.404Z 15 300-319 22 22 22 46 46 46 F F F
27472024-02-27T21:45:03.404Z 16 320-339 22 22 22 46 46 46 T T F <---
27482024-02-27T21:45:03.404Z 17 340-359 19 19 19 39 39 39 F F F
27492024-02-27T21:45:03.404Z 18 360-379 16 16 16 34 34 34 F F F
27502024-02-27T21:45:03.404Z 19 380-399 22 22 22 45 45 45 F F F
27512024-02-27T21:45:03.404Z 20 400-419 22 22 22 45 45 45 F F F
27522024-02-27T21:45:03.405Z 21 420-439 24 24 24 48 48 48 T T F <---
27532024-02-27T21:45:03.405Z 22 440-459 23 23 23 47 47 47 T T F <---
27542024-02-27T21:45:03.405Z 23 460-479 23 23 23 47 47 47 F F F
27552024-02-27T21:45:03.405Z 24 480-499 24 24 23 49 49 47 F F T <---
27562024-02-27T21:45:03.405Z 25 500-519 20 20 20 43 43 43 F F F
27572024-02-27T21:45:03.406Z 26 520-539 22 22 22 46 46 46 F F F
27582024-02-27T21:45:03.406Z 27 540-559 20 20 20 40 40 40 F F F
27592024-02-27T21:45:03.406Z 28 560-579 20 20 20 40 40 40 F F F
27602024-02-27T21:45:03.406Z 29 580-599 13 13 13 28 28 28 F F F
27612024-02-27T21:45:03.406ZMax gen: 24, Max flush: 49
27622024-02-27T21:45:03.406ZError: Difference in extent metadata found!
27632024-02-27T21:45:03.407ZA Difference in extent metadata is expected here
27642024-02-27T21:45:03.407ZOn loop 12
27652024-02-27T21:45:03.407Z
27662024-02-27T21:45:03.407ZVerifying data now
27672024-02-27T21:45:06.187ZNo change after verify
27682024-02-27T21:45:06.187ZLoop: 12 Downstairs dump after verify (and repair):
27692024-02-27T21:45:06.226ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27702024-02-27T21:45:06.226Z 0 000-019 10 10 10 18 18 18 F F F
27712024-02-27T21:45:06.226Z 1 020-039 25 25 25 50 50 50 F F F
27722024-02-27T21:45:06.226Z 2 040-059 25 25 25 50 50 50 F F F
27732024-02-27T21:45:06.226Z 3 060-079 25 25 25 50 50 50 F F F
27742024-02-27T21:45:06.226Z 4 080-099 25 25 25 50 50 50 F F F
27752024-02-27T21:45:06.227Z 5 100-119 22 22 22 45 45 45 F F F
27762024-02-27T21:45:06.227Z 6 120-139 16 16 16 34 34 34 F F F
27772024-02-27T21:45:06.227Z 7 140-159 25 25 25 50 50 50 F F F
27782024-02-27T21:45:06.227Z 8 160-179 25 25 25 50 50 50 F F F
27792024-02-27T21:45:06.227Z 9 180-199 25 25 25 50 50 50 F F F
27802024-02-27T21:45:06.227Z 10 200-219 14 14 14 30 30 30 F F F
27812024-02-27T21:45:06.227Z 11 220-239 24 24 24 48 48 48 F F F
27822024-02-27T21:45:06.227Z 12 240-259 23 23 23 47 47 47 F F F
27832024-02-27T21:45:06.227Z 13 260-279 25 25 25 50 50 50 F F F
27842024-02-27T21:45:06.227Z 14 280-299 20 20 20 40 40 40 F F F
27852024-02-27T21:45:06.227Z 15 300-319 22 22 22 46 46 46 F F F
27862024-02-27T21:45:06.228Z 16 320-339 25 25 25 50 50 50 F F F
27872024-02-27T21:45:06.228Z 17 340-359 19 19 19 39 39 39 F F F
27882024-02-27T21:45:06.228Z 18 360-379 16 16 16 34 34 34 F F F
27892024-02-27T21:45:06.228Z 19 380-399 22 22 22 45 45 45 F F F
27902024-02-27T21:45:06.228Z 20 400-419 22 22 22 45 45 45 F F F
27912024-02-27T21:45:06.228Z 21 420-439 25 25 25 50 50 50 F F F
27922024-02-27T21:45:06.228Z 22 440-459 25 25 25 50 50 50 F F F
27932024-02-27T21:45:06.228Z 23 460-479 23 23 23 47 47 47 F F F
27942024-02-27T21:45:06.228Z 24 480-499 24 24 24 49 49 49 F F F
27952024-02-27T21:45:06.229Z 25 500-519 20 20 20 43 43 43 F F F
27962024-02-27T21:45:06.229Z 26 520-539 22 22 22 46 46 46 F F F
27972024-02-27T21:45:06.229Z 27 540-559 20 20 20 40 40 40 F F F
27982024-02-27T21:45:06.229Z 28 560-579 20 20 20 40 40 40 F F F
27992024-02-27T21:45:06.229Z 29 580-599 13 13 13 28 28 28 F F F
28002024-02-27T21:45:06.229ZMax gen: 25, Max flush: 50
28012024-02-27T21:45:06.229Z
28022024-02-27T21:45:06.230ZBegin loop 13
28032024-02-27T21:45:06.230ZDownstairs to restart: 2
28042024-02-27T21:45:06.837Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.61962602Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242}
28052024-02-27T21:45:06.838Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
28062024-02-27T21:45:06.838Z,"time":"2024-02-27T21:45:04.619980458Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242}
28072024-02-27T21:45:06.838Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620025633Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242}
28082024-02-27T21:45:06.838Z{"msg":"Crucible 67f3a9f9-8b8c-49e4-a511-a287bc9ae030 has session id: 8a6bcf7a-5e33-404a-b036-1fede8672225","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.62005554Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28092024-02-27T21:45:06.838Z{"msg":"Upstairs opts: Upstairs UUID: 67f3a9f9-8b8c-49e4-a511-a287bc9ae030, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620105224Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28102024-02-27T21:45:06.838Z{"msg":"Crucible stats registered with UUID: 67f3a9f9-8b8c-49e4-a511-a287bc9ae030","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620131501Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28112024-02-27T21:45:06.838Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620167787Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242}
28122024-02-27T21:45:06.838Z{"msg":"67f3a9f9-8b8c-49e4-a511-a287bc9ae030 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620185074Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28132024-02-27T21:45:06.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.62022173Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28142024-02-27T21:45:06.838Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620282983Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28152024-02-27T21:45:06.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620341476Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28162024-02-27T21:45:06.838Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620376202Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28172024-02-27T21:45:06.838Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620404789Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28182024-02-27T21:45:06.838Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620448953Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28192024-02-27T21:45:06.838Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620545502Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"io task","client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28202024-02-27T21:45:06.838Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620605125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"io task","client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28212024-02-27T21:45:06.838Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620651879Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"io task","client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28222024-02-27T21:45:06.839Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620679146Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"io task","client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28232024-02-27T21:45:06.839Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620725171Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"io task","client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28242024-02-27T21:45:06.839Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.620743569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"io task","client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28252024-02-27T21:45:06.839Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.621251629Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28262024-02-27T21:45:06.839Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.621301583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28272024-02-27T21:45:06.839Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.621343218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28282024-02-27T21:45:06.839Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.621371425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28292024-02-27T21:45:06.839Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.621397172Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28302024-02-27T21:45:06.839Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.621429118Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28312024-02-27T21:45:06.839Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.621820022Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28322024-02-27T21:45:06.839Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.621872866Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28332024-02-27T21:45:06.839Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622105338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28342024-02-27T21:45:06.839Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622300865Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28352024-02-27T21:45:06.839Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.62234395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28362024-02-27T21:45:06.839Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622360218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28372024-02-27T21:45:06.839Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622454807Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28382024-02-27T21:45:06.839Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622503192Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28392024-02-27T21:45:06.839Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622587182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28402024-02-27T21:45:06.839Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622623287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28412024-02-27T21:45:06.839Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622639805Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28422024-02-27T21:45:06.839Z{"msg":"[0]R flush_numbers[0..12]: [18, 50, 50, 50, 50, 45, 34, 50, 50, 50, 30, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622671812Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28432024-02-27T21:45:06.840Z{"msg":"[0]R generation[0..12]: [10, 25, 25, 25, 25, 22, 16, 25, 25, 25, 14, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622689419Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28442024-02-27T21:45:06.840Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622721256Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28452024-02-27T21:45:06.840Z{"msg":"[1]R flush_numbers[0..12]: [18, 50, 50, 50, 50, 45, 34, 50, 50, 50, 30, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622737224Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28462024-02-27T21:45:06.840Z{"msg":"[1]R generation[0..12]: [10, 25, 25, 25, 25, 22, 16, 25, 25, 25, 14, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.6227689Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28472024-02-27T21:45:06.840Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622795437Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28482024-02-27T21:45:06.840Z{"msg":"Wait for a query_work_queue command to finish before sending IO
28492024-02-27T21:45:06.840Z[2]R flush_numbers[0..12]: [18, 50, 50, 50, 50, 45, 34, 50, 50, 50, 30, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622829723Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28502024-02-27T21:45:06.840Z{"msg":"[2]R generation[0..12]: [10, 25, 25, 25, 25, 22, 16, 25, 25, 25, 14, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622846221Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28512024-02-27T21:45:06.840Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622878867Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28522024-02-27T21:45:06.840Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622907294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28532024-02-27T21:45:06.840Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622925382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28542024-02-27T21:45:06.840Z{"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.6229405Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28552024-02-27T21:45:06.840Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622970726Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"":"downstairs","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28562024-02-27T21:45:06.840Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.622988904Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28572024-02-27T21:45:06.841Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.623016171Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"0","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28582024-02-27T21:45:06.841Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.623031019Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"1","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28592024-02-27T21:45:06.841Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.623045148Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"client":"2","":"downstairs","session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28602024-02-27T21:45:06.841Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.623066245Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28612024-02-27T21:45:06.841Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.623079084Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28622024-02-27T21:45:06.841Z{"msg":"67f3a9f9-8b8c-49e4-a511-a287bc9ae030 is now active with session: 8a6bcf7a-5e33-404a-b036-1fede8672225","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.623098661Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28632024-02-27T21:45:06.841Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.62311113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242,"session_id":"8a6bcf7a-5e33-404a-b036-1fede8672225"}
28642024-02-27T21:45:06.841Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:04.623130798Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242}
28652024-02-27T21:45:06.841ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28662024-02-27T21:45:06.841ZLoading write count information from file "/tmp/test_repair_verify.data"
28672024-02-27T21:45:06.841ZRun Repair workload
28682024-02-27T21:45:06.841Z01/30 Write block 70 len 12288 data: 2 3 4
28692024-02-27T21:45:06.841Z02/30 Read block 359 len 24576
28702024-02-27T21:45:06.841Z03/30 Read block 517 len 24576
28712024-02-27T21:45:06.841Z04/30 Write block 21 len 24576 data: 2 2 2 2 2 2
28722024-02-27T21:45:06.841Z05/30 Write block 355 len 24576 data: 2 2 2 3 4 4
28732024-02-27T21:45:06.842Z06/30 Read block 372 len 32768
28742024-02-27T21:45:06.842Z07/30 Read block 62 len 40960
28752024-02-27T21:45:06.842Z08/30 Read block 254 len 20480
28762024-02-27T21:45:06.842Z09/30 Write block 267 len 12288 data: 3 3 3
28772024-02-27T21:45:06.842Z10/30 Write block 50 len 16384 data: 3 3 3 4
28782024-02-27T21:45:06.842Z11/30 Read block 574 len 8192
28792024-02-27T21:45:06.842Z12/30 Read block 223 len 36864
28802024-02-27T21:45:06.842Z13/30 Write block 392 len 24576 data: 4 5 3 3 3 3
28812024-02-27T21:45:06.842Z14/30 Flush
28822024-02-27T21:45:06.842Z15/30 Write block 105 len 4096 data: 3
28832024-02-27T21:45:06.842Z16/30 Read block 98 len 12288
28842024-02-27T21:45:06.842Z17/30 Read block 204 len 36864
28852024-02-27T21:45:06.842Z18/30 Flush
28862024-02-27T21:45:06.842Z19/30 Write block 123 len 36864 data: 4 3 3 3 3 3 2 2 2
28872024-02-27T21:45:06.842Z20/30 Read block 286 len 36864
28882024-02-27T21:45:06.842Z21/30 Read block 132 len 4096
28892024-02-27T21:45:06.843Z22/30 Read block 29 len 24576
28902024-02-27T21:45:06.843Z23/30 Write block 498 len 40960 data: 5 5 4 2 2 2 2 2 2 2
28912024-02-27T21:45:06.843Z24/30 Write block 21 len 20480 data: 3 3 3 3 3
28922024-02-27T21:45:06.843Z25/30 Write block 96 len 32768 data: 3 3 2 2 3 3 3 3
28932024-02-27T21:45:06.843Z26/30 Write block 12 len 4096 data: 3
28942024-02-27T21:45:06.843Z27/30 Write block 93 len 28672 data: 3 3 3 4 4 3 3
28952024-02-27T21:45:06.843Z28/30 Write block 342 len 28672 data: 5 4 4 3 3 2 3
28962024-02-27T21:45:06.843Z29/30 Read block 178 len 8192
28972024-02-27T21:45:06.843Z30/30 Read block 189 len 8192
28982024-02-27T21:45:06.843Z----------------------------------------------------------------
28992024-02-27T21:45:06.843Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29002024-02-27T21:45:06.843ZStates: Active Active Active
29012024-02-27T21:45:06.843ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29022024-02-27T21:45:06.843Z 1 Acked 1000 Write 3 Done Done Done false
29032024-02-27T21:45:06.843Z 2 Acked 1001 Read 6 Done Done Done false
29042024-02-27T21:45:07.408Z 3 Acked 1002 Read 6 Done Done Done false
29052024-02-27T21:45:07.408Z 4 Acked 1003 Write 6 Done Done Done false
29062024-02-27T21:45:07.408Z 5 Acked 1004 Write 6 Done Done Sent false
29072024-02-27T21:45:07.409Z 6 Acked 1005 Read 8 Done Done Sent false
29082024-02-27T21:45:07.409Z 7 Acked 1006 Read 10 Done Done Sent false
29092024-02-27T21:45:07.409Z 8 Acked 1007 Read 5 Done Done Sent false
29102024-02-27T21:45:07.409Z 9 Acked 1008 Write 3 Done Done Sent false
29112024-02-27T21:45:07.409Z 10 Acked 1009 Write 4 Done Done Sent false
29122024-02-27T21:45:07.409Z 11 Acked 1010 Read 2 Done Done Sent false
29132024-02-27T21:45:07.409Z 12 Acked 1011 Read 9 Done Done Sent false
29142024-02-27T21:45:07.409Z 13 Acked 1012 Write 6 Done Done Sent false
29152024-02-27T21:45:07.410Z 14 Acked 1013 Flush 0 Done Done Sent false
29162024-02-27T21:45:07.410Z 15 Acked 1014 Write 1 Done Done Sent false
29172024-02-27T21:45:07.410Z 16 Acked 1015 Read 3 Done Done Sent false
29182024-02-27T21:45:07.410Z 17 Acked 1016 Read 9 Done Done Sent false
29192024-02-27T21:45:07.410Z 18 Acked 1017 Flush 0 Done Done Sent false
29202024-02-27T21:45:07.410Z 19 Acked 1018 Write 9 Done Done Sent false
29212024-02-27T21:45:07.410Z 20 Acked 1019 Read 9 Done Done Sent false
29222024-02-27T21:45:07.411Z 21 Acked 1020 Read 1 Done Done Sent false
29232024-02-27T21:45:07.411Z 22 Acked 1021 Read 6 Done Done Sent false
29242024-02-27T21:45:07.411Z 23 Acked 1022 Write 10 Done Done Sent false
29252024-02-27T21:45:07.411Z 24 Acked 1023 Write 5 Done Done Sent false
29262024-02-27T21:45:07.411Z 25 Acked 1024 Write 8 Done Done Sent false
29272024-02-27T21:45:07.411Z 26 Acked 1025 Write 1 Done Done Sent false
29282024-02-27T21:45:07.411Z 27 Acked 1026 Write 7 Done Done Sent false
29292024-02-27T21:45:07.412Z 28 Acked 1027 Write 7 Done Done Sent false
29302024-02-27T21:45:07.412Z 29 Acked 1028 Read 2 Done Done Sent false
29312024-02-27T21:45:07.412Z 30 Acked 1029 Read 2 Done Done Sent false
29322024-02-27T21:45:07.412Z STATES DS:0 DS:1 DS:2 TOTAL
29332024-02-27T21:45:07.413Z New 0 0 0 0
29342024-02-27T21:45:07.413Z Sent 0 0 26 26
29352024-02-27T21:45:07.413Z Done 30 30 4 64
29362024-02-27T21:45:07.413Z Skipped 0 0 0 0
29372024-02-27T21:45:07.413Z Error 0 0 0 0
29382024-02-27T21:45:07.414ZLast Flush: 1017 1017 0
29392024-02-27T21:45:07.414ZDownstairs last five completed:
29402024-02-27T21:45:07.414ZUpstairs last five completed: 30 29 28 27 26
29412024-02-27T21:45:07.414Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:04.652187325Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1242}
29422024-02-27T21:45:07.414ZWrote out file "/tmp/test_repair_verify.data"
29432024-02-27T21:45:07.414Z
29442024-02-27T21:45:07.414ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29452024-02-27T21:45:07.415ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29462024-02-27T21:45:07.415Z 0 000-019 10 10 10 18 18 18 T T F <---
29472024-02-27T21:45:07.415Z 1 020-039 26 26 25 51 51 50 T T T <---
29482024-02-27T21:45:07.415Z 2 040-059 26 26 25 51 51 50 F F F <---
29492024-02-27T21:45:07.415Z 3 060-079 26 26 25 51 51 50 F F T <---
29502024-02-27T21:45:07.415Z 4 080-099 25 25 25 50 50 50 T T F <---
29512024-02-27T21:45:07.416Z 5 100-119 26 26 22 52 52 45 T T F <---
29522024-02-27T21:45:07.416Z 6 120-139 16 16 16 34 34 34 T T F <---
29532024-02-27T21:45:07.416Z 7 140-159 25 25 25 50 50 50 F F F
29542024-02-27T21:45:07.416Z 8 160-179 25 25 25 50 50 50 F F F
29552024-02-27T21:45:07.416Z 9 180-199 25 25 25 50 50 50 F F F
29562024-02-27T21:45:07.416Z 10 200-219 14 14 14 30 30 30 F F F
29572024-02-27T21:45:07.417Z 11 220-239 24 24 24 48 48 48 F F F
29582024-02-27T21:45:07.417Z 12 240-259 23 23 23 47 47 47 F F F
29592024-02-27T21:45:07.417Z 13 260-279 26 26 25 51 51 50 F F F <---
29602024-02-27T21:45:07.417Z 14 280-299 20 20 20 40 40 40 F F F
29612024-02-27T21:45:07.417Z 15 300-319 22 22 22 46 46 46 F F F
29622024-02-27T21:45:07.418Z 16 320-339 25 25 25 50 50 50 F F F
29632024-02-27T21:45:07.418Z 17 340-359 26 26 19 51 51 39 T T F <---
29642024-02-27T21:45:07.418Z 18 360-379 26 26 16 51 51 34 F F F <---
29652024-02-27T21:45:07.418Z 19 380-399 26 26 22 51 51 45 F F F <---
29662024-02-27T21:45:07.418Z 20 400-419 22 22 22 45 45 45 F F F
29672024-02-27T21:45:07.418Z 21 420-439 25 25 25 50 50 50 F F F
29682024-02-27T21:45:07.418Z 22 440-459 25 25 25 50 50 50 F F F
29692024-02-27T21:45:07.419Z 23 460-479 23 23 23 47 47 47 F F F
29702024-02-27T21:45:07.419Z 24 480-499 24 24 24 49 49 49 T T F <---
29712024-02-27T21:45:07.419Z 25 500-519 20 20 20 43 43 43 T T F <---
29722024-02-27T21:45:07.419Z 26 520-539 22 22 22 46 46 46 F F F
29732024-02-27T21:45:07.419Z 27 540-559 20 20 20 40 40 40 F F F
29742024-02-27T21:45:07.420Z 28 560-579 20 20 20 40 40 40 F F F
29752024-02-27T21:45:07.420Z 29 580-599 13 13 13 28 28 28 F F F
29762024-02-27T21:45:07.420ZMax gen: 26, Max flush: 52
29772024-02-27T21:45:07.420ZError: Difference in extent metadata found!
29782024-02-27T21:45:07.420ZA Difference in extent metadata is expected here
29792024-02-27T21:45:07.421ZOn loop 13
29802024-02-27T21:45:07.421Z
29812024-02-27T21:45:07.421ZVerifying data now
29822024-02-27T21:45:11.927ZNo change after verify
29832024-02-27T21:45:11.927ZLoop: 13 Downstairs dump after verify (and repair):
29842024-02-27T21:45:11.937ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29852024-02-27T21:45:11.937Z 0 000-019 27 27 27 53 53 53 F F F
29862024-02-27T21:45:11.937Z 1 020-039 27 27 27 53 53 53 F F F
29872024-02-27T21:45:11.937Z 2 040-059 26 26 26 51 51 51 F F F
29882024-02-27T21:45:11.937Z 3 060-079 26 26 26 51 51 51 F F F
29892024-02-27T21:45:11.937Z 4 080-099 27 27 27 53 53 53 F F F
29902024-02-27T21:45:11.949Z 5 100-119 27 27 27 53 53 53 F F F
29912024-02-27T21:45:11.949Z 6 120-139 27 27 27 53 53 53 F F F
29922024-02-27T21:45:11.950Z 7 140-159 25 25 25 50 50 50 F F F
29932024-02-27T21:45:11.950Z 8 160-179 25 25 25 50 50 50 F F F
29942024-02-27T21:45:11.950Z 9 180-199 25 25 25 50 50 50 F F F
29952024-02-27T21:45:11.950Z 10 200-219 14 14 14 30 30 30 F F F
29962024-02-27T21:45:11.950Z 11 220-239 24 24 24 48 48 48 F F F
29972024-02-27T21:45:11.950Z 12 240-259 23 23 23 47 47 47 F F F
29982024-02-27T21:45:11.950Z 13 260-279 26 26 26 51 51 51 F F F
29992024-02-27T21:45:11.950Z 14 280-299 20 20 20 40 40 40 F F F
30002024-02-27T21:45:11.950Z 15 300-319 22 22 22 46 46 46 F F F
30012024-02-27T21:45:11.951Z 16 320-339 25 25 25 50 50 50 F F F
30022024-02-27T21:45:11.951Z 17 340-359 27 27 27 53 53 53 F F F
30032024-02-27T21:45:11.951Z 18 360-379 26 26 26 51 51 51 F F F
30042024-02-27T21:45:11.951Z 19 380-399 26 26 26 51 51 51 F F F
30052024-02-27T21:45:11.951Z 20 400-419 22 22 22 45 45 45 F F F
30062024-02-27T21:45:11.951Z 21 420-439 25 25 25 50 50 50 F F F
30072024-02-27T21:45:11.951Z 22 440-459 25 25 25 50 50 50 F F F
30082024-02-27T21:45:11.951Z 23 460-479 23 23 23 47 47 47 F F F
30092024-02-27T21:45:11.952Z 24 480-499 27 27 27 53 53 53 F F F
30102024-02-27T21:45:11.952Z 25 500-519 27 27 27 53 53 53 F F F
30112024-02-27T21:45:11.952Z 26 520-539 22 22 22 46 46 46 F F F
30122024-02-27T21:45:11.952Z 27 540-559 20 20 20 40 40 40 F F F
30132024-02-27T21:45:11.952Z 28 560-579 20 20 20 40 40 40 F F F
30142024-02-27T21:45:11.952Z 29 580-599 13 13 13 28 28 28 F F F
30152024-02-27T21:45:11.952ZMax gen: 27, Max flush: 53
30162024-02-27T21:45:11.952Z
30172024-02-27T21:45:11.952ZBegin loop 14
30182024-02-27T21:45:11.952ZDownstairs to restart: 2
30192024-02-27T21:45:11.977Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.36457428Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250}
30202024-02-27T21:45:11.977Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
30212024-02-27T21:45:11.977Z,"time":"2024-02-27T21:45:10.36491035Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250}
30222024-02-27T21:45:11.978Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.364944056Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250}
30232024-02-27T21:45:11.978Z{"msg":"Crucible 9c47025b-f214-463d-adae-a24fcdeb5be8 has session id: c3ef900f-a8e9-4403-81c2-812a0333c7fe","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.364983931Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30242024-02-27T21:45:11.978Z{"msg":"Upstairs opts: Upstairs UUID: 9c47025b-f214-463d-adae-a24fcdeb5be8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365024676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30252024-02-27T21:45:11.978Z{"msg":"Crucible stats registered with UUID: 9c47025b-f214-463d-adae-a24fcdeb5be8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365066832Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30262024-02-27T21:45:11.978Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365090359Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250}
30272024-02-27T21:45:11.978Z{"msg":"9c47025b-f214-463d-adae-a24fcdeb5be8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365175769Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30282024-02-27T21:45:11.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365206345Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30292024-02-27T21:45:11.978Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.36525144Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30302024-02-27T21:45:11.979Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365288745Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30312024-02-27T21:45:11.979Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365328831Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30322024-02-27T21:45:11.979Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365355928Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30332024-02-27T21:45:11.979Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365391923Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30342024-02-27T21:45:11.979Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365428709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"io task","client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30352024-02-27T21:45:11.979Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365471544Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"io task","client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30362024-02-27T21:45:11.979Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365623206Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"io task","client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30372024-02-27T21:45:11.979Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365689798Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"io task","client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30382024-02-27T21:45:11.979Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.365718055Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"io task","client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30392024-02-27T21:45:11.979Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.36575509Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"io task","client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30402024-02-27T21:45:11.979Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.366255132Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30412024-02-27T21:45:11.980Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.366293467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30422024-02-27T21:45:11.980Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.366325973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30432024-02-27T21:45:11.980Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.366364529Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30442024-02-27T21:45:11.980Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.366398525Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30452024-02-27T21:45:11.980Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.366427541Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30462024-02-27T21:45:11.980Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.366821785Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30472024-02-27T21:45:11.980Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.366905705Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30482024-02-27T21:45:11.980Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367132898Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30492024-02-27T21:45:11.980Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367447411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30502024-02-27T21:45:11.980Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367489986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30512024-02-27T21:45:11.981Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367513274Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30522024-02-27T21:45:11.981Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367549859Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30532024-02-27T21:45:11.981Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367564467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30542024-02-27T21:45:11.981Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367590604Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30552024-02-27T21:45:11.981Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367605963Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30562024-02-27T21:45:11.981Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.36762576Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30572024-02-27T21:45:11.981Z{"msg":"[0]R flush_numbers[0..12]: [53, 53, 51, 51, 53, 53, 53, 50, 50, 50, 30, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367653907Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30582024-02-27T21:45:11.981Z{"msg":"[0]R generation[0..12]: [27, 27, 26, 26, 27, 27, 27, 25, 25, 25, 14, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367669785Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30592024-02-27T21:45:11.981Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367684813Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30602024-02-27T21:45:11.982Z{"msg":"[1]R flush_numbers[0..12]: [53, 53, 51, 51, 53, 53, 53, 50, 50, 50, 30, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367706071Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30612024-02-27T21:45:11.982Z{"msg":"[1]R generation[0..12]: [27, 27, 26, 26, 27, 27, 27, 25, 25, 25, 14, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367720159Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30622024-02-27T21:45:11.982Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367741316Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30632024-02-27T21:45:11.982Z{"msg":"[2]R flush_numbers[0..12]: [53, 53, 51, 51, 53, 53, 53, 50, 50, 50, 30, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367756035Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30642024-02-27T21:45:11.982Z{"msg":"[2]R generation[0..12]: [27, 27, 26, 26, 27, 27, 27, 25, 25, 25, 14, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367770823Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30652024-02-27T21:45:11.982Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367785111Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30662024-02-27T21:45:11.982Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.36779916Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"Wait for a query_work_queue command to finish before sending IO
30672024-02-27T21:45:11.982Z}
30682024-02-27T21:45:11.982Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367824807Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30692024-02-27T21:45:11.982Z{"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367838085Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30702024-02-27T21:45:11.982Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367859183Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"":"downstairs","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30712024-02-27T21:45:11.983Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.36788091Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30722024-02-27T21:45:11.983Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367893289Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"0","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30732024-02-27T21:45:11.983Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367907037Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"1","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30742024-02-27T21:45:11.983Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367927475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"client":"2","":"downstairs","session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30752024-02-27T21:45:11.983Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367941333Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30762024-02-27T21:45:11.983Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367953951Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30772024-02-27T21:45:11.983Z{"msg":"9c47025b-f214-463d-adae-a24fcdeb5be8 is now active with session: c3ef900f-a8e9-4403-81c2-812a0333c7fe","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.367980918Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30782024-02-27T21:45:11.983Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.368005356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250,"session_id":"c3ef900f-a8e9-4403-81c2-812a0333c7fe"}
30792024-02-27T21:45:11.983Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:10.368018054Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250}
30802024-02-27T21:45:11.984ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30812024-02-27T21:45:11.984ZLoading write count information from file "/tmp/test_repair_verify.data"
30822024-02-27T21:45:11.984ZRun Repair workload
30832024-02-27T21:45:11.990Z01/30 Write block 135 len 12288 data: 3 3 3
30842024-02-27T21:45:11.990Z02/30 Read block 243 len 24576
30852024-02-27T21:45:11.991Z03/30 Write block 554 len 20480 data: 5 5 4 7 6
30862024-02-27T21:45:11.991Z04/30 Write block 580 len 40960 data: 3 3 3 3 3 3 3 3 2 2
30872024-02-27T21:45:11.991Z05/30 Read block 233 len 32768
30882024-02-27T21:45:11.991Z06/30 Write block 536 len 20480 data: 2 2 2 2 2
30892024-02-27T21:45:11.991Z07/30 Read block 43 len 32768
30902024-02-27T21:45:11.991Z08/30 Read block 512 len 24576
30912024-02-27T21:45:11.991Z09/30 Write block 180 len 8192 data: 2 2
30922024-02-27T21:45:11.991Z10/30 Write block 30 len 16384 data: 5 4 3 3
30932024-02-27T21:45:11.991Z11/30 Write block 24 len 28672 data: 4 4 3 3 3 4 6
30942024-02-27T21:45:11.991Z12/30 Read block 350 len 36864
30952024-02-27T21:45:11.991Z13/30 Write block 578 len 20480 data: 5 4 4 4 4
30962024-02-27T21:45:11.992Z14/30 Read block 511 len 8192
30972024-02-27T21:45:11.992Z15/30 Write block 14 len 16384 data: 3 2 2 2
30982024-02-27T21:45:11.992Z16/30 Write block 432 len 36864 data: 4 4 5 3 3 4 3 3 3
30992024-02-27T21:45:11.992Z17/30 Write block 484 len 20480 data: 4 4 2 2 2
31002024-02-27T21:45:11.992Z18/30 Read block 37 len 8192
31012024-02-27T21:45:11.992Z19/30 Read block 350 len 40960
31022024-02-27T21:45:11.992Z20/30 Write block 316 len 36864 data: 5 3 3 3 3 3 2 2 2
31032024-02-27T21:45:11.992Z21/30 Read block 170 len 16384
31042024-02-27T21:45:11.992Z22/30 Write block 84 len 32768 data: 4 4 3 4 5 5 3 2
31052024-02-27T21:45:11.992Z23/30 Read block 15 len 12288
31062024-02-27T21:45:11.992Z24/30 Read block 195 len 8192
31072024-02-27T21:45:11.993Z25/30 Read block 121 len 28672
31082024-02-27T21:45:11.993Z26/30 Read block 421 len 40960
31092024-02-27T21:45:11.993Z27/30 Write block 236 len 4096 data: 6
31102024-02-27T21:45:11.993Z28/30 Read block 335 len 28672
31112024-02-27T21:45:11.993Z29/30 Read block 137 len 20480
31122024-02-27T21:45:11.993Z30/30 Write block 9 len 40960 data: 3 3 4 4 3 4 3 3 3 2
31132024-02-27T21:45:12.006Z----------------------------------------------------------------
31142024-02-27T21:45:12.006Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31152024-02-27T21:45:12.007ZStates: Active Active Active
31162024-02-27T21:45:12.007ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31172024-02-27T21:45:12.007Z 1 Acked 1000 Write 3 Done Done Done false
31182024-02-27T21:45:12.007Z 2 Acked 1001 Read 6 Done Done Done false
31192024-02-27T21:45:12.007Z 3 Acked 1002 Write 5 Done Done Done false
31202024-02-27T21:45:12.008Z 4 Acked 1003 Write 10 Done Done Done false
31212024-02-27T21:45:12.008Z 5 Acked 1004 Read 8 Done Done Done false
31222024-02-27T21:45:12.008Z 6 Acked 1005 Write 5 Done Done Done false
31232024-02-27T21:45:12.008Z 7 Acked 1006 Read 8 Done Done Done false
31242024-02-27T21:45:12.008Z 8 Acked 1007 Read 6 Done Done Done false
31252024-02-27T21:45:12.008Z 9 Acked 1008 Write 2 Done Done Done false
31262024-02-27T21:45:12.009Z 10 Acked 1009 Write 4 Done Done Sent false
31272024-02-27T21:45:12.009Z 11 Acked 1010 Write 7 Done Done Sent false
31282024-02-27T21:45:12.009Z 12 Acked 1011 Read 9 Done Done Sent false
31292024-02-27T21:45:12.010Z 13 Acked 1012 Write 5 Done Done Sent false
31302024-02-27T21:45:12.010Z 14 Acked 1013 Read 2 Done Done Sent false
31312024-02-27T21:45:12.010Z 15 Acked 1014 Write 4 Done Done Sent false
31322024-02-27T21:45:12.010Z 16 Acked 1015 Write 9 Done Done Sent false
31332024-02-27T21:45:12.010Z 17 Acked 1016 Write 5 Done Done Sent false
31342024-02-27T21:45:12.010Z 18 Acked 1017 Read 2 Done Done Sent false
31352024-02-27T21:45:12.010Z 19 Acked 1018 Read 10 Done Done Sent false
31362024-02-27T21:45:12.010Z 20 Acked 1019 Write 9 Done Done Sent false
31372024-02-27T21:45:12.011Z 21 Acked 1020 Read 4 Done Done Sent false
31382024-02-27T21:45:12.011Z 22 Acked 1021 Write 8 Done Done Sent false
31392024-02-27T21:45:12.011Z 23 Acked 1022 Read 3 Done Done Sent false
31402024-02-27T21:45:12.011Z 24 Acked 1023 Read 2 Done Done Sent false
31412024-02-27T21:45:12.011Z 25 Acked 1024 Read 7 Done Done Sent false
31422024-02-27T21:45:12.011Z 26 Acked 1025 Read 10 Done Done Sent false
31432024-02-27T21:45:12.012Z 27 Acked 1026 Write 1 Done Done Sent false
31442024-02-27T21:45:12.012Z 28 Acked 1027 Read 7 Done Done Sent false
31452024-02-27T21:45:12.012Z 29 Acked 1028 Read 5 Done Done Sent false
31462024-02-27T21:45:12.013Z 30 Acked 1029 Write 10 Sent Sent Sent false
31472024-02-27T21:45:12.013Z STATES DS:0 DS:1 DS:2 TOTAL
31482024-02-27T21:45:12.013Z New 0 0 0 0
31492024-02-27T21:45:12.013Z Sent 1 1 21 23
31502024-02-27T21:45:12.018Z Done 29 29 9 67
31512024-02-27T21:45:12.018Z Skipped 0 0 0 0
31522024-02-27T21:45:12.018Z Error 0 0 0 0
31532024-02-27T21:45:12.018ZLast Flush: 0 0 0
31542024-02-27T21:45:12.018ZDownstairs last five completed:
31552024-02-27T21:45:12.019ZUpstairs last five completed: 30 29 28 27 26
31562024-02-27T21:45:12.019Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:10.398623389Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1250}
31572024-02-27T21:45:12.019ZWrote out file "/tmp/test_repair_verify.data"
31582024-02-27T21:45:12.019Z
31592024-02-27T21:45:12.019ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31602024-02-27T21:45:12.046ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31612024-02-27T21:45:12.046Z 0 000-019 27 27 27 53 53 53 T T F <---
31622024-02-27T21:45:12.047Z 1 020-039 27 27 27 53 53 53 T T F <---
31632024-02-27T21:45:12.047Z 2 040-059 26 26 26 51 51 51 F F F
31642024-02-27T21:45:12.047Z 3 060-079 26 26 26 51 51 51 F F F
31652024-02-27T21:45:12.047Z 4 080-099 27 27 27 53 53 53 T T F <---
31662024-02-27T21:45:12.047Z 5 100-119 27 27 27 53 53 53 F F F
31672024-02-27T21:45:12.048Z 6 120-139 27 27 27 53 53 53 T T T
31682024-02-27T21:45:12.048Z 7 140-159 25 25 25 50 50 50 F F F
31692024-02-27T21:45:12.048Z 8 160-179 25 25 25 50 50 50 F F F
31702024-02-27T21:45:12.048Z 9 180-199 25 25 25 50 50 50 T T T
31712024-02-27T21:45:12.048Z 10 200-219 14 14 14 30 30 30 F F F
31722024-02-27T21:45:12.048Z 11 220-239 24 24 24 48 48 48 T T F <---
31732024-02-27T21:45:12.048Z 12 240-259 23 23 23 47 47 47 F F F
31742024-02-27T21:45:12.049Z 13 260-279 26 26 26 51 51 51 F F F
31752024-02-27T21:45:12.049Z 14 280-299 20 20 20 40 40 40 F F F
31762024-02-27T21:45:12.049Z 15 300-319 22 22 22 46 46 46 T T F <---
31772024-02-27T21:45:12.049Z 16 320-339 25 25 25 50 50 50 T T F <---
31782024-02-27T21:45:12.049Z 17 340-359 27 27 27 53 53 53 F F F
31792024-02-27T21:45:12.049Z 18 360-379 26 26 26 51 51 51 F F F
31802024-02-27T21:45:12.049Z 19 380-399 26 26 26 51 51 51 F F F
31812024-02-27T21:45:12.049Z 20 400-419 22 22 22 45 45 45 F F F
31822024-02-27T21:45:12.049Z 21 420-439 25 25 25 50 50 50 T T F <---
31832024-02-27T21:45:12.049Z 22 440-459 25 25 25 50 50 50 T T F <---
31842024-02-27T21:45:12.049Z 23 460-479 23 23 23 47 47 47 F F F
31852024-02-27T21:45:12.050Z 24 480-499 27 27 27 53 53 53 T T F <---
31862024-02-27T21:45:12.050Z 25 500-519 27 27 27 53 53 53 F F F
31872024-02-27T21:45:12.050Z 26 520-539 22 22 22 46 46 46 T T T
31882024-02-27T21:45:12.050Z 27 540-559 20 20 20 40 40 40 T T T
31892024-02-27T21:45:12.050Z 28 560-579 20 20 20 40 40 40 T T F <---
31902024-02-27T21:45:12.050Z 29 580-599 13 13 13 28 28 28 T T T
31912024-02-27T21:45:12.050ZMax gen: 27, Max flush: 53
31922024-02-27T21:45:12.050ZError: Difference in extent metadata found!
31932024-02-27T21:45:12.050ZA Difference in extent metadata is expected here
31942024-02-27T21:45:12.051ZOn loop 14
31952024-02-27T21:45:12.051Z
31962024-02-27T21:45:12.051ZVerifying data now
31972024-02-27T21:45:17.765ZNo change after verify
31982024-02-27T21:45:17.766ZLoop: 14 Downstairs dump after verify (and repair):
31992024-02-27T21:45:17.782ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32002024-02-27T21:45:17.783Z 0 000-019 28 28 28 54 54 54 F F F
32012024-02-27T21:45:17.783Z 1 020-039 28 28 28 54 54 54 F F F
32022024-02-27T21:45:17.783Z 2 040-059 26 26 26 51 51 51 F F F
32032024-02-27T21:45:17.783Z 3 060-079 26 26 26 51 51 51 F F F
32042024-02-27T21:45:17.783Z 4 080-099 28 28 28 54 54 54 F F F
32052024-02-27T21:45:17.784Z 5 100-119 27 27 27 53 53 53 F F F
32062024-02-27T21:45:17.784Z 6 120-139 28 28 28 54 54 54 F F F
32072024-02-27T21:45:17.784Z 7 140-159 25 25 25 50 50 50 F F F
32082024-02-27T21:45:17.784Z 8 160-179 25 25 25 50 50 50 F F F
32092024-02-27T21:45:17.785Z 9 180-199 28 28 28 54 54 54 F F F
32102024-02-27T21:45:17.785Z 10 200-219 14 14 14 30 30 30 F F F
32112024-02-27T21:45:17.785Z 11 220-239 28 28 28 54 54 54 F F F
32122024-02-27T21:45:17.785Z 12 240-259 23 23 23 47 47 47 F F F
32132024-02-27T21:45:17.785Z 13 260-279 26 26 26 51 51 51 F F F
32142024-02-27T21:45:17.785Z 14 280-299 20 20 20 40 40 40 F F F
32152024-02-27T21:45:17.786Z 15 300-319 28 28 28 54 54 54 F F F
32162024-02-27T21:45:17.786Z 16 320-339 28 28 28 54 54 54 F F F
32172024-02-27T21:45:17.787Z 17 340-359 27 27 27 53 53 53 F F F
32182024-02-27T21:45:17.797Z 18 360-379 26 26 26 51 51 51 F F F
32192024-02-27T21:45:17.797Z 19 380-399 26 26 26 51 51 51 F F F
32202024-02-27T21:45:17.797Z 20 400-419 22 22 22 45 45 45 F F F
32212024-02-27T21:45:17.798Z 21 420-439 28 28 28 54 54 54 F F F
32222024-02-27T21:45:17.798Z 22 440-459 28 28 28 54 54 54 F F F
32232024-02-27T21:45:17.798Z 23 460-479 23 23 23 47 47 47 F F F
32242024-02-27T21:45:17.798Z 24 480-499 28 28 28 54 54 54 F F F
32252024-02-27T21:45:17.798Z 25 500-519 27 27 27 53 53 53 F F F
32262024-02-27T21:45:17.798Z 26 520-539 28 28 28 54 54 54 F F F
32272024-02-27T21:45:17.799Z 27 540-559 28 28 28 54 54 54 F F F
32282024-02-27T21:45:17.799Z 28 560-579 28 28 28 54 54 54 F F F
32292024-02-27T21:45:17.799Z 29 580-599 28 28 28 54 54 54 F F F
32302024-02-27T21:45:17.799ZMax gen: 28, Max flush: 54
32312024-02-27T21:45:17.799Z
32322024-02-27T21:45:17.799ZBegin loop 15
32332024-02-27T21:45:17.800ZDownstairs to restart: 2
32342024-02-27T21:45:17.816Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.211712452Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258}
32352024-02-27T21:45:17.816Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
32362024-02-27T21:45:17.817Z,"time":"2024-02-27T21:45:16.212058072Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258}
32372024-02-27T21:45:17.817Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212161759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258}
32382024-02-27T21:45:17.817Z{"msg":"Crucible fe7ba2ba-6d97-442e-b59a-a624b9114e5c has session id: 68730258-03c2-459a-a5ed-8325e940467a","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212194685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32392024-02-27T21:45:17.817Z{"msg":"Upstairs opts: Upstairs UUID: fe7ba2ba-6d97-442e-b59a-a624b9114e5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212219802Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32402024-02-27T21:45:17.817Z{"msg":"Crucible stats registered with UUID: fe7ba2ba-6d97-442e-b59a-a624b9114e5c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.2122441Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32412024-02-27T21:45:17.818Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212258378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258}
32422024-02-27T21:45:17.818Z{"msg":"fe7ba2ba-6d97-442e-b59a-a624b9114e5c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212287125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32432024-02-27T21:45:17.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212307532Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32442024-02-27T21:45:17.818Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.21232649Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32452024-02-27T21:45:17.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212340688Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32462024-02-27T21:45:17.818Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212354717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32472024-02-27T21:45:17.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212377524Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32482024-02-27T21:45:17.818Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.21241166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32492024-02-27T21:45:17.819Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212433237Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"io task","client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32502024-02-27T21:45:17.819Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212471973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"io task","client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32512024-02-27T21:45:17.819Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.21249674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"io task","client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32522024-02-27T21:45:17.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212727083Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"io task","client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32532024-02-27T21:45:17.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212776207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"io task","client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32542024-02-27T21:45:17.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.212807543Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"io task","client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32552024-02-27T21:45:17.820Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.21343065Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32562024-02-27T21:45:17.820Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.213468486Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32572024-02-27T21:45:17.820Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.213484334Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32582024-02-27T21:45:17.820Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.213576683Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32592024-02-27T21:45:17.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214129378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32602024-02-27T21:45:17.821Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214174972Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32612024-02-27T21:45:17.821Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214211688Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32622024-02-27T21:45:17.821Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214252803Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32632024-02-27T21:45:17.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214845623Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32642024-02-27T21:45:17.837Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214886119Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32652024-02-27T21:45:17.837Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214901577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32662024-02-27T21:45:17.837Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214919335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32672024-02-27T21:45:17.838Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214945212Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32682024-02-27T21:45:17.838Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.214965479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32692024-02-27T21:45:17.838Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215003525Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32702024-02-27T21:45:17.838Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215393529Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32712024-02-27T21:45:17.838Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215446742Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32722024-02-27T21:45:17.838Z{"msg":"[0]R flush_numbers[0..12]: [54, 54, 51, 51, 54, 53, 54, 50, 50, 54, 30, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215485098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32732024-02-27T21:45:17.838Z{"msg":"[0]R generation[0..12]: [28, 28, 26, 26, 28, 27, 28, 25, 25, 28, 14, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215511395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32742024-02-27T21:45:17.838Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215536692Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32752024-02-27T21:45:17.839Z{"msg":"[1]R flush_numbers[0..12]: [54, 54, 51, 51, 54, 53, 54, 50, 50, 54, 30, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215563459Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32762024-02-27T21:45:17.839Z{"msg":"Wait for a query_work_queue command to finish before sending IO
32772024-02-27T21:45:17.839Z[1]R generation[0..12]: [28, 28, 26, 26, 28, 27, 28, 25, 25, 28, 14, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215601794Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32782024-02-27T21:45:17.839Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215644519Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32792024-02-27T21:45:17.839Z{"msg":"[2]R flush_numbers[0..12]: [54, 54, 51, 51, 54, 53, 54, 50, 50, 54, 30, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215678885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32802024-02-27T21:45:17.839Z{"msg":"[2]R generation[0..12]: [28, 28, 26, 26, 28, 27, 28, 25, 25, 28, 14, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215716151Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32812024-02-27T21:45:17.840Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215754856Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32822024-02-27T21:45:17.840Z{"msg":"Max found gen is 29","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215791582Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32832024-02-27T21:45:17.840Z{"msg":"Generation requested: 30 >= found:29","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215821798Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32842024-02-27T21:45:17.840Z{"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215840276Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32852024-02-27T21:45:17.840Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215863863Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"":"downstairs","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32862024-02-27T21:45:17.840Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215900309Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32872024-02-27T21:45:17.841Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.215935835Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"0","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32882024-02-27T21:45:17.841Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.21597356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
32892024-02-27T21:45:17.841Z,"pid":1258,"client":"1","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32902024-02-27T21:45:17.841Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.216016375Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"client":"2","":"downstairs","session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32912024-02-27T21:45:17.841Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.216043492Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32922024-02-27T21:45:17.841Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.21605697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32932024-02-27T21:45:17.842Z{"msg":"fe7ba2ba-6d97-442e-b59a-a624b9114e5c is now active with session: 68730258-03c2-459a-a5ed-8325e940467a","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.216070049Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32942024-02-27T21:45:17.842Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.216089647Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258,"session_id":"68730258-03c2-459a-a5ed-8325e940467a"}
32952024-02-27T21:45:17.842Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:16.216102595Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258}
32962024-02-27T21:45:17.842ZLoading write count information from file "/tmp/test_repair_verify.data"
32972024-02-27T21:45:17.842ZRun Repair workload
32982024-02-27T21:45:17.842Z01/30 Write block 480 len 36864 data: 4 5 5 5 5 5 3 3 3
32992024-02-27T21:45:17.842Z02/30 Write block 350 len 40960 data: 3 2 2 2 2 3 3 3 4 5
33002024-02-27T21:45:17.842Z03/30 Write block 175 len 16384 data: 4 3 3 3
33012024-02-27T21:45:17.842Z04/30 Write block 377 len 20480 data: 2 2 2 2 2
33022024-02-27T21:45:17.843Z05/30 Write block 150 len 4096 data: 4
33032024-02-27T21:45:17.843Z06/30 Read block 320 len 24576
33042024-02-27T21:45:17.843Z07/30 Write block 139 len 20480 data: 3 3 3 3 3
33052024-02-27T21:45:17.843Z08/30 Read block 40 len 16384
33062024-02-27T21:45:17.843Z09/30 Read block 178 len 32768
33072024-02-27T21:45:17.843Z10/30 Write block 429 len 4096 data: 4
33082024-02-27T21:45:17.843Z11/30 Write block 107 len 20480 data: 4 4 4 3 4
33092024-02-27T21:45:17.843Z12/30 Flush
33102024-02-27T21:45:17.843Z13/30 Write block 387 len 20480 data: 5 5 6 5 5
33112024-02-27T21:45:17.843Z14/30 Read block 430 len 24576
33122024-02-27T21:45:17.844Z15/30 Write block 479 len 12288 data: 3 5 6
33132024-02-27T21:45:17.844Z16/30 Write block 6 len 24576 data: 3 3 3 4 4 5
33142024-02-27T21:45:17.844Z17/30 Write block 518 len 32768 data: 2 2 2 2 3 2 2 2
33152024-02-27T21:45:17.844Z18/30 Write block 34 len 40960 data: 4 3 3 3 4 5 5 5 4 5
33162024-02-27T21:45:17.844Z19/30 Read block 216 len 40960
33172024-02-27T21:45:17.844Z20/30 Read block 431 len 24576
33182024-02-27T21:45:17.844Z21/30 Read block 427 len 12288
33192024-02-27T21:45:17.866Z22/30 Read block 364 len 20480
33202024-02-27T21:45:17.866Z23/30 Write block 158 len 36864 data: 6 6 5 5 4 4 3 2 2
33212024-02-27T21:45:17.866Z24/30 Read block 98 len 32768
33222024-02-27T21:45:17.867Z25/30 Write block 347 len 40960 data: 3 4 3 4 3 3 3 3 4 4
33232024-02-27T21:45:17.867Z26/30 Read block 383 len 28672
33242024-02-27T21:45:17.868Z27/30 Write block 52 len 24576 data: 4 5 4 3 5 5
33252024-02-27T21:45:17.868Z28/30 Read block 502 len 12288
33262024-02-27T21:45:17.868Z29/30 Write block 439 len 4096 data: 4
33272024-02-27T21:45:17.868Z30/30 Read block 562 len 24576
33282024-02-27T21:45:17.868Z----------------------------------------------------------------
33292024-02-27T21:45:17.868Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33302024-02-27T21:45:17.868ZStates: Active Active Active
33312024-02-27T21:45:17.868ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33322024-02-27T21:45:17.869Z 1 Acked 1000 Write 9 Done Done Sent false
33332024-02-27T21:45:17.869Z 2 Acked 1001 Write 10 Done Done Sent false
33342024-02-27T21:45:17.869Z 3 Acked 1002 Write 4 Done Done Sent false
33352024-02-27T21:45:17.869Z 4 Acked 1003 Write 5 Done Done Sent false
33362024-02-27T21:45:17.869Z 5 Acked 1004 Write 1 Done Done Sent false
33372024-02-27T21:45:17.870Z 6 Acked 1005 Read 6 Done Done Sent false
33382024-02-27T21:45:17.870Z 7 Acked 1006 Write 5 Done Done Sent false
33392024-02-27T21:45:17.870Z 8 Acked 1007 Read 4 Done Done Sent false
33402024-02-27T21:45:17.870Z 9 Acked 1008 Read 8 Done Done Sent false
33412024-02-27T21:45:17.871Z 10 Acked 1009 Write 1 Done Done Sent false
33422024-02-27T21:45:17.871Z 11 Acked 1010 Write 5 Done Done Sent false
33432024-02-27T21:45:17.871Z 12 Acked 1011 Flush 0 Done Done Sent false
33442024-02-27T21:45:17.871Z 13 Acked 1012 Write 5 Done Done Sent false
33452024-02-27T21:45:17.871Z 14 Acked 1013 Read 6 Done Done Sent false
33462024-02-27T21:45:17.871Z 15 Acked 1014 Write 3 Done Done Sent false
33472024-02-27T21:45:17.871Z 16 Acked 1015 Write 6 Done Done Sent false
33482024-02-27T21:45:17.871Z 17 Acked 1016 Write 8 Done Done Sent false
33492024-02-27T21:45:17.872Z 18 Acked 1017 Write 10 Done Done Sent false
33502024-02-27T21:45:17.872Z 19 Acked 1018 Read 10 Done Done Sent false
33512024-02-27T21:45:17.872Z 20 Acked 1019 Read 6 Done Done Sent false
33522024-02-27T21:45:17.872Z 21 Acked 1020 Read 3 Done Done Sent false
33532024-02-27T21:45:17.872Z 22 Acked 1021 Read 5 Done Done Sent false
33542024-02-27T21:45:17.872Z 23 Acked 1022 Write 9 Done Done Sent false
33552024-02-27T21:45:17.872Z 24 Acked 1023 Read 8 Done Done Sent false
33562024-02-27T21:45:17.873Z 25 Acked 1024 Write 10 Done Done Sent false
33572024-02-27T21:45:17.873Z 26 Acked 1025 Read 7 Done Done Sent false
33582024-02-27T21:45:17.873Z 27 Acked 1026 Write 6 Done Done Sent false
33592024-02-27T21:45:17.873Z 28 Acked 1027 Read 3 Done Done Sent false
33602024-02-27T21:45:17.873Z 29 Acked 1028 Write 1 Done Done Sent false
33612024-02-27T21:45:17.873Z 30 Acked 1029 Read 6 Done Done Sent false
33622024-02-27T21:45:17.873Z STATES DS:0 DS:1 DS:2 TOTAL
33632024-02-27T21:45:17.873Z New 0 0 0 0
33642024-02-27T21:45:17.874Z Sent 0 0 30 30
33652024-02-27T21:45:17.874Z Done 30 30 0 60
33662024-02-27T21:45:17.874Z Skipped 0 0 0 0
33672024-02-27T21:45:17.874Z Error 0 0 0 0
33682024-02-27T21:45:17.874ZLast Flush: 1011 1011 0
33692024-02-27T21:45:17.875ZDownstairs last five completed:
33702024-02-27T21:45:17.875ZUpstairs last five completed: 30 29 28 27 26
33712024-02-27T21:45:17.875Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:16.246183732Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1258}
33722024-02-27T21:45:17.875ZWrote out file "/tmp/test_repair_verify.data"
33732024-02-27T21:45:17.875Z
33742024-02-27T21:45:17.875ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33752024-02-27T21:45:17.886ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33762024-02-27T21:45:17.886Z 0 000-019 28 28 28 54 54 54 T T F <---
33772024-02-27T21:45:17.886Z 1 020-039 28 28 28 54 54 54 T T F <---
33782024-02-27T21:45:17.886Z 2 040-059 26 26 26 51 51 51 T T F <---
33792024-02-27T21:45:17.887Z 3 060-079 26 26 26 51 51 51 F F F
33802024-02-27T21:45:17.887Z 4 080-099 28 28 28 54 54 54 F F F
33812024-02-27T21:45:17.887Z 5 100-119 30 30 27 55 55 53 F F F <---
33822024-02-27T21:45:17.887Z 6 120-139 30 30 28 55 55 54 F F F <---
33832024-02-27T21:45:17.887Z 7 140-159 30 30 25 55 55 50 T T F <---
33842024-02-27T21:45:17.887Z 8 160-179 30 30 25 55 55 50 T T F <---
33852024-02-27T21:45:17.887Z 9 180-199 28 28 28 54 54 54 F F F
33862024-02-27T21:45:17.887Z 10 200-219 14 14 14 30 30 30 F F F
33872024-02-27T21:45:17.887Z 11 220-239 28 28 28 54 54 54 F F F
33882024-02-27T21:45:17.887Z 12 240-259 23 23 23 47 47 47 F F F
33892024-02-27T21:45:17.888Z 13 260-279 26 26 26 51 51 51 F F F
33902024-02-27T21:45:17.888Z 14 280-299 20 20 20 40 40 40 F F F
33912024-02-27T21:45:17.888Z 15 300-319 28 28 28 54 54 54 F F F
33922024-02-27T21:45:17.888Z 16 320-339 28 28 28 54 54 54 F F F
33932024-02-27T21:45:17.888Z 17 340-359 30 30 27 55 55 53 T T F <---
33942024-02-27T21:45:17.888Z 18 360-379 30 30 26 55 55 51 F F F <---
33952024-02-27T21:45:17.888Z 19 380-399 30 30 26 55 55 51 T T F <---
33962024-02-27T21:45:17.888Z 20 400-419 22 22 22 45 45 45 F F F
33972024-02-27T21:45:17.888Z 21 420-439 30 30 28 55 55 54 T T F <---
33982024-02-27T21:45:17.889Z 22 440-459 28 28 28 54 54 54 F F F
33992024-02-27T21:45:17.889Z 23 460-479 23 23 23 47 47 47 T T F <---
34002024-02-27T21:45:17.889Z 24 480-499 30 30 28 55 55 54 T T F <---
34012024-02-27T21:45:17.889Z 25 500-519 27 27 27 53 53 53 T T F <---
34022024-02-27T21:45:17.889Z 26 520-539 28 28 28 54 54 54 T T F <---
34032024-02-27T21:45:17.889Z 27 540-559 28 28 28 54 54 54 F F F
34042024-02-27T21:45:17.889Z 28 560-579 28 28 28 54 54 54 F F F
34052024-02-27T21:45:17.890Z 29 580-599 28 28 28 54 54 54 F F F
34062024-02-27T21:45:17.890ZMax gen: 30, Max flush: 55
34072024-02-27T21:45:17.890ZError: Difference in extent metadata found!
34082024-02-27T21:45:17.890ZA Difference in extent metadata is expected here
34092024-02-27T21:45:17.890ZOn loop 15
34102024-02-27T21:45:17.890Z
34112024-02-27T21:45:17.891ZVerifying data now
34122024-02-27T21:45:22.892ZNo change after verify
34132024-02-27T21:45:22.892ZLoop: 15 Downstairs dump after verify (and repair):
34142024-02-27T21:45:23.340ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34152024-02-27T21:45:23.340Z 0 000-019 31 31 31 56 56 56 F F F
34162024-02-27T21:45:23.340Z 1 020-039 31 31 31 56 56 56 F F F
34172024-02-27T21:45:23.340Z 2 040-059 31 31 31 56 56 56 F F F
34182024-02-27T21:45:23.341Z 3 060-079 26 26 26 51 51 51 F F F
34192024-02-27T21:45:23.341Z 4 080-099 28 28 28 54 54 54 F F F
34202024-02-27T21:45:23.341Z 5 100-119 30 30 30 55 55 55 F F F
34212024-02-27T21:45:23.341Z 6 120-139 30 30 30 55 55 55 F F F
34222024-02-27T21:45:23.341Z 7 140-159 31 31 31 56 56 56 F F F
34232024-02-27T21:45:23.341Z 8 160-179 31 31 31 56 56 56 F F F
34242024-02-27T21:45:23.341Z 9 180-199 28 28 28 54 54 54 F F F
34252024-02-27T21:45:23.341Z 10 200-219 14 14 14 30 30 30 F F F
34262024-02-27T21:45:23.341Z 11 220-239 28 28 28 54 54 54 F F F
34272024-02-27T21:45:23.342Z 12 240-259 23 23 23 47 47 47 F F F
34282024-02-27T21:45:23.342Z 13 260-279 26 26 26 51 51 51 F F F
34292024-02-27T21:45:23.342Z 14 280-299 20 20 20 40 40 40 F F F
34302024-02-27T21:45:23.342Z 15 300-319 28 28 28 54 54 54 F F F
34312024-02-27T21:45:23.342Z 16 320-339 28 28 28 54 54 54 F F F
34322024-02-27T21:45:23.342Z 17 340-359 31 31 31 56 56 56 F F F
34332024-02-27T21:45:23.342Z 18 360-379 30 30 30 55 55 55 F F F
34342024-02-27T21:45:23.342Z 19 380-399 31 31 31 56 56 56 F F F
34352024-02-27T21:45:23.342Z 20 400-419 22 22 22 45 45 45 F F F
34362024-02-27T21:45:23.342Z 21 420-439 31 31 31 56 56 56 F F F
34372024-02-27T21:45:23.343Z 22 440-459 28 28 28 54 54 54 F F F
34382024-02-27T21:45:23.343Z 23 460-479 31 31 31 56 56 56 F F F
34392024-02-27T21:45:23.343Z 24 480-499 31 31 31 56 56 56 F F F
34402024-02-27T21:45:23.343Z 25 500-519 31 31 31 56 56 56 F F F
34412024-02-27T21:45:23.343Z 26 520-539 31 31 31 56 56 56 F F F
34422024-02-27T21:45:23.343Z 27 540-559 28 28 28 54 54 54 F F F
34432024-02-27T21:45:23.343Z 28 560-579 28 28 28 54 54 54 F F F
34442024-02-27T21:45:23.343Z 29 580-599 28 28 28 54 54 54 F F F
34452024-02-27T21:45:23.343ZMax gen: 31, Max flush: 56
34462024-02-27T21:45:23.343Z
34472024-02-27T21:45:23.343ZBegin loop 16
34482024-02-27T21:45:23.344ZDownstairs to restart: 2
34492024-02-27T21:45:23.344Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.05644361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266}
34502024-02-27T21:45:23.344Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.05669992Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266}
34512024-02-27T21:45:23.344Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.056738875Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266}
34522024-02-27T21:45:23.344Z{"msg":"Crucible 00fd2510-6818-4f95-8946-ef6d42b1567c has session id: c17efe73-f320-4993-a341-89a28764c94d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.056761512Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34532024-02-27T21:45:23.344ZCrucible runtime is spawned
34542024-02-27T21:45:23.344Z{"msg":"Upstairs opts: Upstairs UUID: 00fd2510-6818-4f95-8946-ef6d42b1567c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.056817356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34552024-02-27T21:45:23.344Z{"msg":"Crucible stats registered with UUID: 00fd2510-6818-4f95-8946-ef6d42b1567c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.056851112Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34562024-02-27T21:45:23.344Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.05686784Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266}
34572024-02-27T21:45:23.344Z{"msg":"00fd2510-6818-4f95-8946-ef6d42b1567c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.056971098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34582024-02-27T21:45:23.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.056997965Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34592024-02-27T21:45:23.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057028871Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34602024-02-27T21:45:23.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057066756Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34612024-02-27T21:45:23.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057092304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34622024-02-27T21:45:23.345Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057129449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34632024-02-27T21:45:23.345Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057157446Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34642024-02-27T21:45:23.345Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057191882Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"io task","client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34652024-02-27T21:45:23.345Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057219489Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"io task","client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34662024-02-27T21:45:23.345Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057271992Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"io task","client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34672024-02-27T21:45:23.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057435233Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"io task","client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34682024-02-27T21:45:23.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057488017Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"io task","client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34692024-02-27T21:45:23.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.057519553Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"io task","client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34702024-02-27T21:45:23.345Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.058205502Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34712024-02-27T21:45:23.345Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.058244058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34722024-02-27T21:45:23.345Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.058259686Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34732024-02-27T21:45:23.345Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.058274304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34742024-02-27T21:45:23.345Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.058316409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34752024-02-27T21:45:23.345Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.058364644Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34762024-02-27T21:45:23.345Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.058788314Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34772024-02-27T21:45:23.345Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.058843637Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34782024-02-27T21:45:23.345Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.058979131Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34792024-02-27T21:45:23.346Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059362926Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34802024-02-27T21:45:23.346Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.05941375Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34812024-02-27T21:45:23.346Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059472893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34822024-02-27T21:45:23.346Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059509489Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34832024-02-27T21:45:23.346Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059526337Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34842024-02-27T21:45:23.346Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059540505Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34852024-02-27T21:45:23.346Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059553654Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34862024-02-27T21:45:23.346Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059567062Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34872024-02-27T21:45:23.346Z{"msg":"[0]R flush_numbers[0..12]: [56, 56, 56, 51, 54, 55, 55, 56, 56, 54, 30, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059588799Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34882024-02-27T21:45:23.346Z{"msg":"[0]R generation[0..12]: [31, 31, 31, 26, 28, 30, 30, 31, 31, 28, 14, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059603368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34892024-02-27T21:45:23.346Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059618276Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34902024-02-27T21:45:23.346Z{"msg":"[1]R flush_numbers[0..12]: [56, 56, 56, 51, 54, 55, 55, 56, 56, 54, 30, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059632684Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34912024-02-27T21:45:23.346Z{"msg":"[1]R generation[0..12]: [31, 31, 31, 26, 28, 30, 30, 31, 31, 28, 14, 28]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
34922024-02-27T21:45:23.346Z2024-02-27T21:45:21.059653232Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34932024-02-27T21:45:23.347Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.05967118Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34942024-02-27T21:45:23.347Z{"msg":"[2]R flush_numbers[0..12]: [56, 56, 56, 51, 54, 55, 55, 56, 56, 54, 30, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059694507Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34952024-02-27T21:45:23.347Z{"msg":"[2]R generation[0..12]: [31, 31, 31, 26, 28, 30, 30, 31, 31, 28, 14, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059727893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34962024-02-27T21:45:23.347Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059744511Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34972024-02-27T21:45:23.347Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059764769Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34982024-02-27T21:45:23.347Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059779707Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
34992024-02-27T21:45:23.347Z{"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059808484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35002024-02-27T21:45:23.347Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059825482Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35012024-02-27T21:45:23.347Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059847179Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35022024-02-27T21:45:23.347Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059859598Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35032024-02-27T21:45:23.347Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059879585Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35042024-02-27T21:45:23.347Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059893104Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35052024-02-27T21:45:23.347Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059912911Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35062024-02-27T21:45:23.347Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.05992592Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35072024-02-27T21:45:23.348Z{"msg":"00fd2510-6818-4f95-8946-ef6d42b1567c is now active with session: c17efe73-f320-4993-a341-89a28764c94d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059940538Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35082024-02-27T21:45:23.348Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059959316Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266,"session_id":"c17efe73-f320-4993-a341-89a28764c94d"}
35092024-02-27T21:45:23.348Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:21.059971864Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266}
35102024-02-27T21:45:23.348ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35112024-02-27T21:45:23.348ZLoading write count information from file "/tmp/test_repair_verify.data"
35122024-02-27T21:45:23.348ZRun Repair workload
35132024-02-27T21:45:23.348Z01/30 Write block 276 len 20480 data: 2 2 2 2 2
35142024-02-27T21:45:23.378Z02/30 Read block 158 len 16384
35152024-02-27T21:45:23.379Z03/30 Read block 162 len 20480
35162024-02-27T21:45:23.379Z04/30 Write block 366 len 32768 data: 3 3 3 2 3 3 2 2
35172024-02-27T21:45:23.379Z05/30 Read block 535 len 28672
35182024-02-27T21:45:23.379Z06/30 Read block 132 len 40960
35192024-02-27T21:45:23.380Z07/30 Read block 538 len 32768
35202024-02-27T21:45:23.380Z08/30 Write block 363 len 12288 data: 3 3 3
35212024-02-27T21:45:23.380Z09/30 Flush
35222024-02-27T21:45:23.380Z10/30 Write block 467 len 4096 data: 3
35232024-02-27T21:45:23.380Z11/30 Flush
35242024-02-27T21:45:23.380Z12/30 Write block 459 len 32768 data: 4 3 4 5 5 4 4 4
35252024-02-27T21:45:23.381Z13/30 Write block 363 len 36864 data: 4 4 4 4 4 4 3 4 4
35262024-02-27T21:45:23.381Z14/30 Read block 93 len 4096
35272024-02-27T21:45:23.381Z15/30 Read block 348 len 36864
35282024-02-27T21:45:23.381Z16/30 Read block 365 len 28672
35292024-02-27T21:45:23.381Z17/30 Read block 37 len 28672
35302024-02-27T21:45:23.381Z18/30 Write block 248 len 12288 data: 4 4 4
35312024-02-27T21:45:23.382Z19/30 Write block 276 len 28672 data: 3 3 3 3 3 2 2
35322024-02-27T21:45:23.382Z20/30 Write block 247 len 4096 data: 5
35332024-02-27T21:45:23.382Z21/30 Flush
35342024-02-27T21:45:23.382Z22/30 Write block 327 len 36864 data: 3 4 4 4 4 5 5 5 6
35352024-02-27T21:45:23.382Z23/30 Read block 398 len 36864
35362024-02-27T21:45:23.382Z24/30 Read block 308 len 16384
35372024-02-27T21:45:23.382Z25/30 Read block 244 len 28672
35382024-02-27T21:45:23.382Z26/30 Write block 304 len 40960 data: 4 3 4 6 6 7 7 7 6 5
35392024-02-27T21:45:23.382Z27/30 Write block 319 len 8192 data: 4 4
35402024-02-27T21:45:23.382Z28/30 Read block 291 len 8192
35412024-02-27T21:45:23.382Z29/30 Write block 314 len 4096 data: 5
35422024-02-27T21:45:23.382Z30/30 Read block 264 len 24576
35432024-02-27T21:45:23.382Z----------------------------------------------------------------
35442024-02-27T21:45:23.382Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35452024-02-27T21:45:23.383ZStates: Active Active Active
35462024-02-27T21:45:23.383ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35472024-02-27T21:45:23.383Z 1 Acked 1000 Write 5 Done Done Done false
35482024-02-27T21:45:23.383Z 2 Acked 1001 Read 4 Done Done Done false
35492024-02-27T21:45:23.383Z 3 Acked 1002 Read 5 Done Done Done false
35502024-02-27T21:45:23.383Z 4 Acked 1003 Write 8 Done Done Done false
35512024-02-27T21:45:23.383Z 5 Acked 1004 Read 7 Done Done Done false
35522024-02-27T21:45:23.383Z 6 Acked 1005 Read 10 Done Done Done false
35532024-02-27T21:45:23.383Z 7 Acked 1006 Read 8 Done Done Sent false
35542024-02-27T21:45:23.383Z 8 Acked 1007 Write 3 Done Done Sent false
35552024-02-27T21:45:23.383Z 9 Acked 1008 Flush 0 Done Done Sent false
35562024-02-27T21:45:23.384Z 10 Acked 1009 Write 1 Done Done Sent false
35572024-02-27T21:45:23.384Z 11 Acked 1010 Flush 0 Done Done Sent false
35582024-02-27T21:45:23.384Z 12 Acked 1011 Write 8 Done Done Sent false
35592024-02-27T21:45:23.384Z 13 Acked 1012 Write 9 Done Done Sent false
35602024-02-27T21:45:23.384Z 14 Acked 1013 Read 1 Done Done Sent false
35612024-02-27T21:45:23.384Z 15 Acked 1014 Read 9 Done Done Sent false
35622024-02-27T21:45:23.384Z 16 Acked 1015 Read 7 Done Done Sent false
35632024-02-27T21:45:23.384Z 17 Acked 1016 Read 7 Done Done Sent false
35642024-02-27T21:45:23.384Z 18 Acked 1017 Write 3 Done Done Sent false
35652024-02-27T21:45:23.384Z 19 Acked 1018 Write 7 Done Done Sent false
35662024-02-27T21:45:23.384Z 20 Acked 1019 Write 1 Done Done Sent false
35672024-02-27T21:45:23.384Z 21 Acked 1020 Flush 0 Done Done Sent false
35682024-02-27T21:45:23.384Z 22 Acked 1021 Write 9 Done Done Sent false
35692024-02-27T21:45:23.384Z 23 Acked 1022 Read 9 Done Done Sent false
35702024-02-27T21:45:23.384Z 24 Acked 1023 Read 4 Done Done Sent false
35712024-02-27T21:45:23.384Z 25 Acked 1024 Read 7 Done Done Sent false
35722024-02-27T21:45:23.384Z 26 Acked 1025 Write 10 Done Done Sent false
35732024-02-27T21:45:23.385Z 27 Acked 1026 Write 2 Done Done Sent false
35742024-02-27T21:45:23.385Z 28 Acked 1027 Read 2 Done Done Sent false
35752024-02-27T21:45:23.385Z 29 Acked 1028 Write 1 Done Done Sent false
35762024-02-27T21:45:23.385Z 30 Acked 1029 Read 6 Done Done Sent false
35772024-02-27T21:45:23.385Z STATES DS:0 DS:1 DS:2 TOTAL
35782024-02-27T21:45:23.385Z New 0 0 0 0
35792024-02-27T21:45:23.385Z Sent 0 0 24 24
35802024-02-27T21:45:23.385Z Done 30 30 6 66
35812024-02-27T21:45:23.385Z Skipped 0 0 0 0
35822024-02-27T21:45:23.385Z Error 0 0 0 0
35832024-02-27T21:45:23.385ZLast Flush: 1020 1020 0
35842024-02-27T21:45:23.385ZDownstairs last five completed:
35852024-02-27T21:45:23.385ZUpstairs last five completed: 30 29 28 27 26
35862024-02-27T21:45:23.385Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:21.091297204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1266}
35872024-02-27T21:45:23.385ZWrote out file "/tmp/test_repair_verify.data"
35882024-02-27T21:45:23.386Z
35892024-02-27T21:45:23.386ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35902024-02-27T21:45:23.386ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35912024-02-27T21:45:23.386Z 0 000-019 31 31 31 56 56 56 F F F
35922024-02-27T21:45:23.386Z 1 020-039 31 31 31 56 56 56 F F F
35932024-02-27T21:45:23.386Z 2 040-059 31 31 31 56 56 56 F F F
35942024-02-27T21:45:23.386Z 3 060-079 26 26 26 51 51 51 F F F
35952024-02-27T21:45:23.386Z 4 080-099 28 28 28 54 54 54 F F F
35962024-02-27T21:45:23.386Z 5 100-119 30 30 30 55 55 55 F F F
35972024-02-27T21:45:23.386Z 6 120-139 30 30 30 55 55 55 F F F
35982024-02-27T21:45:23.386Z 7 140-159 31 31 31 56 56 56 F F F
35992024-02-27T21:45:23.386Z 8 160-179 31 31 31 56 56 56 F F F
36002024-02-27T21:45:23.386Z 9 180-199 28 28 28 54 54 54 F F F
36012024-02-27T21:45:23.386Z 10 200-219 14 14 14 30 30 30 F F F
36022024-02-27T21:45:23.386Z 11 220-239 28 28 28 54 54 54 F F F
36032024-02-27T21:45:23.386Z 12 240-259 32 32 23 59 59 47 F F F <---
36042024-02-27T21:45:23.387Z 13 260-279 32 32 26 59 59 51 F F T <---
36052024-02-27T21:45:23.387Z 14 280-299 32 32 20 59 59 40 F F T <---
36062024-02-27T21:45:23.387Z 15 300-319 28 28 28 54 54 54 T T F <---
36072024-02-27T21:45:23.387Z 16 320-339 28 28 28 54 54 54 T T F <---
36082024-02-27T21:45:23.387Z 17 340-359 31 31 31 56 56 56 F F F
36092024-02-27T21:45:23.387Z 18 360-379 32 32 30 59 59 55 F F T <---
36102024-02-27T21:45:23.387Z 19 380-399 31 31 31 56 56 56 F F F
36112024-02-27T21:45:23.387Z 20 400-419 22 22 22 45 45 45 F F F
36122024-02-27T21:45:23.387Z 21 420-439 31 31 31 56 56 56 F F F
36132024-02-27T21:45:23.387Z 22 440-459 32 32 28 59 59 54 F F F <---
36142024-02-27T21:45:23.403Z 23 460-479 32 32 31 59 59 56 F F F <---
36152024-02-27T21:45:23.403Z 24 480-499 31 31 31 56 56 56 F F F
36162024-02-27T21:45:23.403Z 25 500-519 31 31 31 56 56 56 F F F
36172024-02-27T21:45:23.403Z 26 520-539 31 31 31 56 56 56 F F F
36182024-02-27T21:45:23.403Z 27 540-559 28 28 28 54 54 54 F F F
36192024-02-27T21:45:23.403Z 28 560-579 28 28 28 54 54 54 F F F
36202024-02-27T21:45:23.403Z 29 580-599 28 28 28 54 54 54 F F F
36212024-02-27T21:45:23.403ZMax gen: 32, Max flush: 59
36222024-02-27T21:45:23.404ZError: Difference in extent metadata found!
36232024-02-27T21:45:23.404ZA Difference in extent metadata is expected here
36242024-02-27T21:45:23.404ZOn loop 16
36252024-02-27T21:45:23.404Z
36262024-02-27T21:45:23.404ZVerifying data now
36272024-02-27T21:45:28.161ZNo change after verify
36282024-02-27T21:45:28.161ZLoop: 16 Downstairs dump after verify (and repair):
36292024-02-27T21:45:28.591ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36302024-02-27T21:45:28.591Z 0 000-019 31 31 31 56 56 56 F F F
36312024-02-27T21:45:28.591Z 1 020-039 31 31 31 56 56 56 F F F
36322024-02-27T21:45:28.591Z 2 040-059 31 31 31 56 56 56 F F F
36332024-02-27T21:45:28.591Z 3 060-079 26 26 26 51 51 51 F F F
36342024-02-27T21:45:28.591Z 4 080-099 28 28 28 54 54 54 F F F
36352024-02-27T21:45:28.592Z 5 100-119 30 30 30 55 55 55 F F F
36362024-02-27T21:45:28.592Z 6 120-139 30 30 30 55 55 55 F F F
36372024-02-27T21:45:28.592Z 7 140-159 31 31 31 56 56 56 F F F
36382024-02-27T21:45:28.592Z 8 160-179 31 31 31 56 56 56 F F F
36392024-02-27T21:45:28.592Z 9 180-199 28 28 28 54 54 54 F F F
36402024-02-27T21:45:28.592Z 10 200-219 14 14 14 30 30 30 F F F
36412024-02-27T21:45:28.592Z 11 220-239 28 28 28 54 54 54 F F F
36422024-02-27T21:45:28.593Z 12 240-259 32 32 32 59 59 59 F F F
36432024-02-27T21:45:28.593Z 13 260-279 32 32 32 59 59 59 F F F
36442024-02-27T21:45:28.593Z 14 280-299 32 32 32 59 59 59 F F F
36452024-02-27T21:45:28.593Z 15 300-319 33 33 33 60 60 60 F F F
36462024-02-27T21:45:28.593Z 16 320-339 33 33 33 60 60 60 F F F
36472024-02-27T21:45:28.593Z 17 340-359 31 31 31 56 56 56 F F F
36482024-02-27T21:45:28.593Z 18 360-379 32 32 32 59 59 59 F F F
36492024-02-27T21:45:28.594Z 19 380-399 31 31 31 56 56 56 F F F
36502024-02-27T21:45:28.594Z 20 400-419 22 22 22 45 45 45 F F F
36512024-02-27T21:45:28.594Z 21 420-439 31 31 31 56 56 56 F F F
36522024-02-27T21:45:28.594Z 22 440-459 32 32 32 59 59 59 F F F
36532024-02-27T21:45:28.594Z 23 460-479 32 32 32 59 59 59 F F F
36542024-02-27T21:45:28.594Z 24 480-499 31 31 31 56 56 56 F F F
36552024-02-27T21:45:28.594Z 25 500-519 31 31 31 56 56 56 F F F
36562024-02-27T21:45:28.595Z 26 520-539 31 31 31 56 56 56 F F F
36572024-02-27T21:45:28.595Z 27 540-559 28 28 28 54 54 54 F F F
36582024-02-27T21:45:28.595Z 28 560-579 28 28 28 54 54 54 F F F
36592024-02-27T21:45:28.595Z 29 580-599 28 28 28 54 54 54 F F F
36602024-02-27T21:45:28.595ZMax gen: 33, Max flush: 60
36612024-02-27T21:45:28.595Z
36622024-02-27T21:45:28.595ZBegin loop 17
36632024-02-27T21:45:28.595ZDownstairs to restart: 1
36642024-02-27T21:45:28.596Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.593480749Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274}
36652024-02-27T21:45:28.596Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.593791372Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274}
36662024-02-27T21:45:28.596Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
36672024-02-27T21:45:28.596Z,"time":"2024-02-27T21:45:26.593816289Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274}
36682024-02-27T21:45:28.596Z{"msg":"Crucible 4ca0c188-9bac-4fd3-8011-1d7d3bbf8df3 has session id: 02bf550a-68dd-4c74-b6e8-089e952fa8d1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.593858025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36692024-02-27T21:45:28.596Z{"msg":"Upstairs opts: Upstairs UUID: 4ca0c188-9bac-4fd3-8011-1d7d3bbf8df3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.593926047Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36702024-02-27T21:45:28.596Z{"msg":"Crucible stats registered with UUID: 4ca0c188-9bac-4fd3-8011-1d7d3bbf8df3","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.593959233Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36712024-02-27T21:45:28.596Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.593975671Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274}
36722024-02-27T21:45:28.597Z{"msg":"4ca0c188-9bac-4fd3-8011-1d7d3bbf8df3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594057921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36732024-02-27T21:45:28.597Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594081088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36742024-02-27T21:45:28.597Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594112985Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36752024-02-27T21:45:28.597Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594158009Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36762024-02-27T21:45:28.597Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594205514Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36772024-02-27T21:45:28.597Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594226031Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36782024-02-27T21:45:28.597Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594246579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36792024-02-27T21:45:28.598Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594276265Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"io task","client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36802024-02-27T21:45:28.598Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594307532Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"io task","client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36812024-02-27T21:45:28.598Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594343127Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"io task","client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36822024-02-27T21:45:28.598Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594369024Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"io task","client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36832024-02-27T21:45:28.598Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.59440546Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"io task","client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36842024-02-27T21:45:28.598Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.594440696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"io task","client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36852024-02-27T21:45:28.598Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.595159461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36862024-02-27T21:45:28.598Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.595198177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36872024-02-27T21:45:28.598Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.595309434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36882024-02-27T21:45:28.598Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.59534094Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36892024-02-27T21:45:28.599Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.595504651Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36902024-02-27T21:45:28.599Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.595554215Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36912024-02-27T21:45:28.599Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.595746582Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36922024-02-27T21:45:28.599Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.595949138Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36932024-02-27T21:45:28.599Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.59619115Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36942024-02-27T21:45:28.599Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596348291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36952024-02-27T21:45:28.599Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596397776Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36962024-02-27T21:45:28.599Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596424622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36972024-02-27T21:45:28.599Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596458248Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36982024-02-27T21:45:28.599Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596491645Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
36992024-02-27T21:45:28.599Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.59652632Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37002024-02-27T21:45:28.599Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596658485Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37012024-02-27T21:45:28.599Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596705029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37022024-02-27T21:45:28.599Z{"msg":"[0]R flush_numbers[0..12]: [56, 56, 56, 51, 54, 55, 55, 56, 56, 54, 30, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596744505Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37032024-02-27T21:45:28.599Z{"msg":"[0]R generation[0..12]: [31, 31, 31, 26, 28, 30, 30, 31, 31, 28, 14, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596764942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37042024-02-27T21:45:28.599Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596796878Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37052024-02-27T21:45:28.599Z{"msg":"[1]R flush_numbers[0..12]: [56, 56, 56, 51, 54, 55, 55, 56, 56, 54, 30, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596814276Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37062024-02-27T21:45:28.599Z{"msg":"[1]R generation[0..12]: [31, 31, 31, 26, 28, 30, 30, 31, 31, 28, 14, 28]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
37072024-02-27T21:45:28.600Z,"level":30,"time":"2024-02-27T21:45:26.596847663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37082024-02-27T21:45:28.600Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.59686564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37092024-02-27T21:45:28.600Z{"msg":"[2]R flush_numbers[0..12]: [56, 56, 56, 51, 54, 55, 55, 56, 56, 54, 30, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596897167Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37102024-02-27T21:45:28.600Z{"msg":"[2]R generation[0..12]: [31, 31, 31, 26, 28, 30, 30, 31, 31, 28, 14, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596913595Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37112024-02-27T21:45:28.600Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596945661Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37122024-02-27T21:45:28.600Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596962919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37132024-02-27T21:45:28.600Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.596993205Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37142024-02-27T21:45:28.600Z{"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597016812Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37152024-02-27T21:45:28.600Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597046209Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37162024-02-27T21:45:28.600Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597061917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37172024-02-27T21:45:28.600Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597091094Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37182024-02-27T21:45:28.600Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597107872Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37192024-02-27T21:45:28.600Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37202024-02-27T21:45:28.600Z,"time":"2024-02-27T21:45:26.59712591Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37212024-02-27T21:45:28.600Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597155526Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37222024-02-27T21:45:28.600Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597174664Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37232024-02-27T21:45:28.600Z{"msg":"4ca0c188-9bac-4fd3-8011-1d7d3bbf8df3 is now active with session: 02bf550a-68dd-4c74-b6e8-089e952fa8d1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597202641Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37242024-02-27T21:45:28.600Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597220939Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274,"session_id":"02bf550a-68dd-4c74-b6e8-089e952fa8d1"}
37252024-02-27T21:45:28.600Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:26.597248235Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274}
37262024-02-27T21:45:28.600ZLoading write count information from file "/tmp/test_repair_verify.data"
37272024-02-27T21:45:28.601ZRun Repair workload
37282024-02-27T21:45:28.601Z01/30 Write block 477 len 40960 data: 2 3 4 6 7 6 6 6 6 4
37292024-02-27T21:45:28.632Z02/30 Read block 231 len 4096
37302024-02-27T21:45:28.632Z03/30 Read block 251 len 32768
37312024-02-27T21:45:28.633Z04/30 Write block 258 len 20480 data: 3 3 4 4 3
37322024-02-27T21:45:28.633Z05/30 Read block 420 len 36864
37332024-02-27T21:45:28.633Z06/30 Write block 208 len 20480 data: 4 4 4 4 4
37342024-02-27T21:45:28.633Z07/30 Write block 270 len 28672 data: 3 3 2 2 2 2 4
37352024-02-27T21:45:28.633Z08/30 Read block 241 len 24576
37362024-02-27T21:45:28.633Z09/30 Write block 343 len 40960 data: 5 5 4 4 4 5 4 5 4 4
37372024-02-27T21:45:28.633Z10/30 Write block 138 len 32768 data: 3 4 4 4 4 4 3 2
37382024-02-27T21:45:28.634Z11/30 Read block 533 len 16384
37392024-02-27T21:45:28.634Z12/30 Flush
37402024-02-27T21:45:28.634Z13/30 Write block 433 len 16384 data: 5 6 4 4
37412024-02-27T21:45:28.634Z14/30 Write block 384 len 16384 data: 2 3 3 6
37422024-02-27T21:45:28.634Z15/30 Write block 104 len 40960 data: 3 4 4 5 5 5 4 5 4 3
37432024-02-27T21:45:28.635Z16/30 Write block 149 len 16384 data: 4 5 4 4
37442024-02-27T21:45:28.635Z17/30 Read block 122 len 40960
37452024-02-27T21:45:28.635Z18/30 Write block 200 len 24576 data: 3 3 3 3 3 3
37462024-02-27T21:45:28.635Z19/30 Read block 511 len 40960
37472024-02-27T21:45:28.635Z20/30 Read block 505 len 4096
37482024-02-27T21:45:28.636Z21/30 Read block 580 len 12288
37492024-02-27T21:45:28.636Z22/30 Write block 409 len 4096 data: 3
37502024-02-27T21:45:28.636Z23/30 Read block 509 len 4096
37512024-02-27T21:45:28.636Z24/30 Write block 228 len 8192 data: 8 8
37522024-02-27T21:45:28.636Z25/30 Read block 230 len 8192
37532024-02-27T21:45:28.637Z26/30 Read block 256 len 40960
37542024-02-27T21:45:28.637Z27/30 Write block 548 len 16384 data: 3 3 3 4
37552024-02-27T21:45:28.637Z28/30 Write block 449 len 24576 data: 5 6 6 5 5 4
37562024-02-27T21:45:28.637Z29/30 Write block 360 len 4096 data: 5
37572024-02-27T21:45:28.637Z30/30 Write block 282 len 20480 data: 3 3 2 2 3
37582024-02-27T21:45:28.638Z----------------------------------------------------------------
37592024-02-27T21:45:28.638Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37602024-02-27T21:45:28.638ZStates: Active Active Active
37612024-02-27T21:45:28.638ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37622024-02-27T21:45:28.638Z 1 Acked 1000 Write 10 Done Sent Done false
37632024-02-27T21:45:28.638Z 2 Acked 1001 Read 1 Done Sent Done false
37642024-02-27T21:45:28.639Z 3 Acked 1002 Read 8 Done Sent Done false
37652024-02-27T21:45:28.639Z 4 Acked 1003 Write 5 Done Sent Done false
37662024-02-27T21:45:28.639Z 5 Acked 1004 Read 9 Done Sent Done false
37672024-02-27T21:45:28.639Z 6 Acked 1005 Write 5 Done Sent Done false
37682024-02-27T21:45:28.639Z 7 Acked 1006 Write 7 Done Sent Done false
37692024-02-27T21:45:28.639Z 8 Acked 1007 Read 6 Done Sent Done false
37702024-02-27T21:45:28.639Z 9 Acked 1008 Write 10 Done Sent Done false
37712024-02-27T21:45:28.639Z 10 Acked 1009 Write 8 Done Sent Done false
37722024-02-27T21:45:28.640Z 11 Acked 1010 Read 4 Done Sent Done false
37732024-02-27T21:45:28.640Z 12 Acked 1011 Flush 0 Done Sent Done false
37742024-02-27T21:45:28.640Z 13 Acked 1012 Write 4 Done Sent Done false
37752024-02-27T21:45:28.640Z 14 Acked 1013 Write 4 Done Sent Done false
37762024-02-27T21:45:28.640Z 15 Acked 1014 Write 10 Done Sent Done false
37772024-02-27T21:45:28.640Z 16 Acked 1015 Write 4 Done Sent Done false
37782024-02-27T21:45:28.640Z 17 Acked 1016 Read 10 Done Sent Done false
37792024-02-27T21:45:28.640Z 18 Acked 1017 Write 6 Done Sent Done false
37802024-02-27T21:45:28.641Z 19 Acked 1018 Read 10 Done Sent Done false
37812024-02-27T21:45:28.641Z 20 Acked 1019 Read 1 Done Sent Done false
37822024-02-27T21:45:28.641Z 21 Acked 1020 Read 3 Done Sent Done false
37832024-02-27T21:45:28.641Z 22 Acked 1021 Write 1 Done Sent Done false
37842024-02-27T21:45:28.641Z 23 Acked 1022 Read 1 Done Sent Done false
37852024-02-27T21:45:28.641Z 24 Acked 1023 Write 2 Done Sent Done false
37862024-02-27T21:45:28.641Z 25 Acked 1024 Read 2 Done Sent Done false
37872024-02-27T21:45:28.642Z 26 Acked 1025 Read 10 Done Sent Done false
37882024-02-27T21:45:28.642Z 27 Acked 1026 Write 4 Done Sent Done false
37892024-02-27T21:45:28.642Z 28 Acked 1027 Write 6 Done Sent Done false
37902024-02-27T21:45:28.642Z 29 Acked 1028 Write 1 Done Sent Done false
37912024-02-27T21:45:28.642Z 30 Acked 1029 Write 5 Sent Sent Sent false
37922024-02-27T21:45:28.642Z STATES DS:0 DS:1 DS:2 TOTAL
37932024-02-27T21:45:28.642Z New 0 0 0 0
37942024-02-27T21:45:28.643Z Sent 1 30 1 32
37952024-02-27T21:45:28.643Z Done 29 0 29 58
37962024-02-27T21:45:28.643Z Skipped 0 0 0 0
37972024-02-27T21:45:28.643Z Error 0 0 0 0
37982024-02-27T21:45:28.643ZLast Flush: 1011 0 1011
37992024-02-27T21:45:28.644ZDownstairs last five completed:
38002024-02-27T21:45:28.644ZUpstairs last five completed: 30 29 28 27 26
38012024-02-27T21:45:28.644Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:26.625447824Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1274}
38022024-02-27T21:45:28.644ZWrote out file "/tmp/test_repair_verify.data"
38032024-02-27T21:45:28.644Z
38042024-02-27T21:45:28.644ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38052024-02-27T21:45:28.644ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38062024-02-27T21:45:28.645Z 0 000-019 31 31 31 56 56 56 F F F
38072024-02-27T21:45:28.645Z 1 020-039 31 31 31 56 56 56 F F F
38082024-02-27T21:45:28.645Z 2 040-059 31 31 31 56 56 56 F F F
38092024-02-27T21:45:28.645Z 3 060-079 26 26 26 51 51 51 F F F
38102024-02-27T21:45:28.645Z 4 080-099 28 28 28 54 54 54 F F F
38112024-02-27T21:45:28.646Z 5 100-119 30 30 30 55 55 55 T F T <---
38122024-02-27T21:45:28.646Z 6 120-139 34 30 34 61 55 61 F F F <---
38132024-02-27T21:45:28.646Z 7 140-159 34 31 34 61 56 61 T F T <---
38142024-02-27T21:45:28.646Z 8 160-179 31 31 31 56 56 56 F F F
38152024-02-27T21:45:28.646Z 9 180-199 28 28 28 54 54 54 F F F
38162024-02-27T21:45:28.646Z 10 200-219 34 14 34 61 30 61 T F T <---
38172024-02-27T21:45:28.647Z 11 220-239 28 28 28 54 54 54 T F T <---
38182024-02-27T21:45:28.647Z 12 240-259 34 32 34 61 59 61 F F F <---
38192024-02-27T21:45:28.647Z 13 260-279 34 32 34 61 59 61 F F F <---
38202024-02-27T21:45:28.647Z 14 280-299 32 32 32 59 59 59 T F T <---
38212024-02-27T21:45:28.647Z 15 300-319 33 33 33 60 60 60 F F F
38222024-02-27T21:45:28.648Z 16 320-339 33 33 33 60 60 60 F F F
38232024-02-27T21:45:28.648Z 17 340-359 34 31 34 61 56 61 F F F <---
38242024-02-27T21:45:28.648Z 18 360-379 32 32 32 59 59 59 T F T <---
38252024-02-27T21:45:28.648Z 19 380-399 31 31 31 56 56 56 T F T <---
38262024-02-27T21:45:28.648Z 20 400-419 22 22 22 45 45 45 T F T <---
38272024-02-27T21:45:28.648Z 21 420-439 31 31 31 56 56 56 T F T <---
38282024-02-27T21:45:28.648Z 22 440-459 32 32 32 59 59 59 T F T <---
38292024-02-27T21:45:28.659Z 23 460-479 34 32 34 61 59 61 F F F <---
38302024-02-27T21:45:28.660Z 24 480-499 34 31 34 61 56 61 F F F <---
38312024-02-27T21:45:28.660Z 25 500-519 31 31 31 56 56 56 F F F
38322024-02-27T21:45:28.660Z 26 520-539 31 31 31 56 56 56 F F F
38332024-02-27T21:45:28.660Z 27 540-559 28 28 28 54 54 54 T F T <---
38342024-02-27T21:45:28.660Z 28 560-579 28 28 28 54 54 54 F F F
38352024-02-27T21:45:28.660Z 29 580-599 28 28 28 54 54 54 F F F
38362024-02-27T21:45:28.661ZMax gen: 34, Max flush: 61
38372024-02-27T21:45:28.661ZError: Difference in extent metadata found!
38382024-02-27T21:45:28.661ZA Difference in extent metadata is expected here
38392024-02-27T21:45:28.661ZOn loop 17
38402024-02-27T21:45:28.661Z
38412024-02-27T21:45:28.661ZVerifying data now
38422024-02-27T21:45:33.076ZNo change after verify
38432024-02-27T21:45:33.076ZLoop: 17 Downstairs dump after verify (and repair):
38442024-02-27T21:45:33.640ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38452024-02-27T21:45:33.640Z 0 000-019 31 31 31 56 56 56 F F F
38462024-02-27T21:45:33.640Z 1 020-039 31 31 31 56 56 56 F F F
38472024-02-27T21:45:33.641Z 2 040-059 31 31 31 56 56 56 F F F
38482024-02-27T21:45:33.641Z 3 060-079 26 26 26 51 51 51 F F F
38492024-02-27T21:45:33.641Z 4 080-099 28 28 28 54 54 54 F F F
38502024-02-27T21:45:33.641Z 5 100-119 35 35 35 62 62 62 F F F
38512024-02-27T21:45:33.641Z 6 120-139 34 34 34 61 61 61 F F F
38522024-02-27T21:45:33.641Z 7 140-159 35 35 35 62 62 62 F F F
38532024-02-27T21:45:33.641Z 8 160-179 31 31 31 56 56 56 F F F
38542024-02-27T21:45:33.641Z 9 180-199 28 28 28 54 54 54 F F F
38552024-02-27T21:45:33.641Z 10 200-219 35 35 35 62 62 62 F F F
38562024-02-27T21:45:33.642Z 11 220-239 35 35 35 62 62 62 F F F
38572024-02-27T21:45:33.642Z 12 240-259 34 34 34 61 61 61 F F F
38582024-02-27T21:45:33.642Z 13 260-279 34 34 34 61 61 61 F F F
38592024-02-27T21:45:33.642Z 14 280-299 35 35 35 62 62 62 F F F
38602024-02-27T21:45:33.642Z 15 300-319 33 33 33 60 60 60 F F F
38612024-02-27T21:45:33.642Z 16 320-339 33 33 33 60 60 60 F F F
38622024-02-27T21:45:33.642Z 17 340-359 34 34 34 61 61 61 F F F
38632024-02-27T21:45:33.642Z 18 360-379 35 35 35 62 62 62 F F F
38642024-02-27T21:45:33.642Z 19 380-399 35 35 35 62 62 62 F F F
38652024-02-27T21:45:33.642Z 20 400-419 35 35 35 62 62 62 F F F
38662024-02-27T21:45:33.642Z 21 420-439 35 35 35 62 62 62 F F F
38672024-02-27T21:45:33.643Z 22 440-459 35 35 35 62 62 62 F F F
38682024-02-27T21:45:33.643Z 23 460-479 34 34 34 61 61 61 F F F
38692024-02-27T21:45:33.643Z 24 480-499 34 34 34 61 61 61 F F F
38702024-02-27T21:45:33.643Z 25 500-519 31 31 31 56 56 56 F F F
38712024-02-27T21:45:33.643Z 26 520-539 31 31 31 56 56 56 F F F
38722024-02-27T21:45:33.643Z 27 540-559 35 35 35 62 62 62 F F F
38732024-02-27T21:45:33.643Z 28 560-579 28 28 28 54 54 54 F F F
38742024-02-27T21:45:33.643Z 29 580-599 28 28 28 54 54 54 F F F
38752024-02-27T21:45:33.644ZMax gen: 35, Max flush: 62
38762024-02-27T21:45:33.657Z
38772024-02-27T21:45:33.657ZBegin loop 18
38782024-02-27T21:45:33.657ZDownstairs to restart: 2
38792024-02-27T21:45:33.657Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.516493916Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282}
38802024-02-27T21:45:33.658Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.516824747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282}
38812024-02-27T21:45:33.658Z{"msg":"Crucible runtime is spawned
38822024-02-27T21:45:33.658ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.516871551Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282}
38832024-02-27T21:45:33.658Z{"msg":"Crucible 843771cc-4aed-4b51-b82f-14686a87f615 has session id: a6899ba5-2217-4b2f-874d-3297cbb117a2","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.516896858Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38842024-02-27T21:45:33.658Z{"msg":"Upstairs opts: Upstairs UUID: 843771cc-4aed-4b51-b82f-14686a87f615, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.516947622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38852024-02-27T21:45:33.658Z{"msg":"Crucible stats registered with UUID: 843771cc-4aed-4b51-b82f-14686a87f615","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.5169686Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38862024-02-27T21:45:33.658Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.516990607Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282}
38872024-02-27T21:45:33.658Z{"msg":"843771cc-4aed-4b51-b82f-14686a87f615 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517064748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38882024-02-27T21:45:33.658Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517091505Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38892024-02-27T21:45:33.658Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517112103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38902024-02-27T21:45:33.658Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517169636Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38912024-02-27T21:45:33.658Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517200532Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38922024-02-27T21:45:33.658Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517237748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38932024-02-27T21:45:33.658Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517272114Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38942024-02-27T21:45:33.658Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517298391Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"io task","client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38952024-02-27T21:45:33.658Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517342056Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"io task","client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38962024-02-27T21:45:33.658Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517379231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"io task","client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38972024-02-27T21:45:33.658Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517414677Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"io task","client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38982024-02-27T21:45:33.658Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.517511956Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"io task","client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
38992024-02-27T21:45:33.658Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.51755902Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"io task","client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39002024-02-27T21:45:33.658Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.518181417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39012024-02-27T21:45:33.658Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.518217852Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39022024-02-27T21:45:33.659Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.51824026Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39032024-02-27T21:45:33.659Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.518277565Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39042024-02-27T21:45:33.659Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.518318791Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39052024-02-27T21:45:33.659Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.518355276Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39062024-02-27T21:45:33.659Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.518764108Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39072024-02-27T21:45:33.659Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.518908411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39082024-02-27T21:45:33.659Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.518951366Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39092024-02-27T21:45:33.659Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519277708Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39102024-02-27T21:45:33.659Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519317843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39112024-02-27T21:45:33.659Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.51934518Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39122024-02-27T21:45:33.659Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519460346Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39132024-02-27T21:45:33.659Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519505891Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39142024-02-27T21:45:33.659Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519539917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39152024-02-27T21:45:33.659Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519563754Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39162024-02-27T21:45:33.659Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.5195973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39172024-02-27T21:45:33.659Z{"msg":"[0]R flush_numbers[0..12]: [56, 56, 56, 51, 54, 62, 61, 62, 56, 54, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519632396Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39182024-02-27T21:45:33.659Z{"msg":"[0]R generation[0..12]: [31, 31, 31, 26, 28, 35, 34, 35, 31, 28, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519668212Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
39192024-02-27T21:45:33.659Z,"pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39202024-02-27T21:45:33.659Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519702078Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39212024-02-27T21:45:33.659Z{"msg":"[1]R flush_numbers[0..12]: [56, 56, 56, 51, 54, 62, 61, 62, 56, 54, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519740513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39222024-02-27T21:45:33.659Z{"msg":"[1]R generation[0..12]: [31, 31, 31, 26, 28, 35, 34, 35, 31, 28, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519777039Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39232024-02-27T21:45:33.659Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519815204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39242024-02-27T21:45:33.659Z{"msg":"[2]R flush_numbers[0..12]: [56, 56, 56, 51, 54, 62, 61, 62, 56, 54, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.51985509Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39252024-02-27T21:45:33.659Z{"msg":"[2]R generation[0..12]: [31, 31, 31, 26, 28, 35, 34, 35, 31, 28, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519893315Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39262024-02-27T21:45:33.660Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519923632Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39272024-02-27T21:45:33.660Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519960417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39282024-02-27T21:45:33.660Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.519996803Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39292024-02-27T21:45:33.660Z{"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.52002421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39302024-02-27T21:45:33.660Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.520060766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39312024-02-27T21:45:33.660Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.520098491Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39322024-02-27T21:45:33.660ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39332024-02-27T21:45:33.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.520140006Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39342024-02-27T21:45:33.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.520158844Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39352024-02-27T21:45:33.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.520172842Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39362024-02-27T21:45:33.660Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.52019484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39372024-02-27T21:45:33.660Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.520207888Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39382024-02-27T21:45:33.660Z{"msg":"843771cc-4aed-4b51-b82f-14686a87f615 is now active with session: a6899ba5-2217-4b2f-874d-3297cbb117a2","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.520221456Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39392024-02-27T21:45:33.660Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.520240054Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282,"session_id":"a6899ba5-2217-4b2f-874d-3297cbb117a2"}
39402024-02-27T21:45:33.660Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:31.520252573Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282}
39412024-02-27T21:45:33.660ZLoading write count information from file "/tmp/test_repair_verify.data"
39422024-02-27T21:45:33.660ZRun Repair workload
39432024-02-27T21:45:33.660Z01/30 Write block 465 len 12288 data: 5 5 4
39442024-02-27T21:45:33.660Z02/30 Write block 138 len 36864 data: 4 5 5 5 5 5 4 3 4
39452024-02-27T21:45:33.660Z03/30 Write block 14 len 28672 data: 5 4 4 4 3 2 3
39462024-02-27T21:45:33.660Z04/30 Read block 315 len 20480
39472024-02-27T21:45:33.660Z05/30 Read block 203 len 20480
39482024-02-27T21:45:33.660Z06/30 Read block 184 len 24576
39492024-02-27T21:45:33.660Z07/30 Flush
39502024-02-27T21:45:33.660Z08/30 Write block 541 len 20480 data: 2 2 2 2 2
39512024-02-27T21:45:33.660Z09/30 Read block 365 len 40960
39522024-02-27T21:45:33.661Z10/30 Read block 532 len 8192
39532024-02-27T21:45:33.661Z11/30 Read block 59 len 32768
39542024-02-27T21:45:33.661Z12/30 Read block 24 len 28672
39552024-02-27T21:45:33.661Z13/30 Write block 349 len 32768 data: 5 6 5 5 4 4 5 5
39562024-02-27T21:45:33.661Z14/30 Flush
39572024-02-27T21:45:33.661Z15/30 Write block 364 len 32768 data: 5 5 5 5 5 4 5 5
39582024-02-27T21:45:33.661Z16/30 Write block 89 len 36864 data: 6 4 3 2 4 4 4 5 5
39592024-02-27T21:45:33.661Z17/30 Write block 548 len 16384 data: 4 4 4 5
39602024-02-27T21:45:33.661Z18/30 Write block 218 len 36864 data: 5 4 4 4 6 5 6 4 5
39612024-02-27T21:45:33.661Z19/30 Write block 68 len 12288 data: 3 2 3
39622024-02-27T21:45:33.661Z20/30 Read block 487 len 24576
39632024-02-27T21:45:33.661Z21/30 Read block 476 len 40960
39642024-02-27T21:45:33.661Z22/30 Write block 43 len 12288 data: 6 5 4
39652024-02-27T21:45:33.661Z23/30 Read block 96 len 20480
39662024-02-27T21:45:33.661Z24/30 Read block 428 len 24576
39672024-02-27T21:45:33.661Z25/30 Read block 185 len 24576
39682024-02-27T21:45:33.661Z26/30 Write block 502 len 40960 data: 3 3 3 3 3 3 2 3 4 5
39692024-02-27T21:45:33.661Z27/30 Write block 186 len 12288 data: 4 4 6
39702024-02-27T21:45:33.661Z28/30 Write block 106 len 24576 data: 5 6 6 6 5 6
39712024-02-27T21:45:33.661Z29/30 Write block 252 len 16384 data: 4 4 3 3
39722024-02-27T21:45:33.661Z30/30 Read block 410 len 32768
39732024-02-27T21:45:33.661Z----------------------------------------------------------------
39742024-02-27T21:45:33.661Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
39752024-02-27T21:45:33.661ZStates: Active Active Active
39762024-02-27T21:45:33.667ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39772024-02-27T21:45:33.667Z 1 Acked 1000 Write 3 Done Done Done false
39782024-02-27T21:45:33.667Z 2 Acked 1001 Write 9 Done Done Done false
39792024-02-27T21:45:33.667Z 3 Acked 1002 Write 7 Done Done Sent false
39802024-02-27T21:45:33.668Z 4 Acked 1003 Read 5 Done Done Sent false
39812024-02-27T21:45:33.668Z 5 Acked 1004 Read 5 Done Done Sent false
39822024-02-27T21:45:33.668Z 6 Acked 1005 Read 6 Done Done Sent false
39832024-02-27T21:45:33.668Z 7 Acked 1006 Flush 0 Done Done Sent false
39842024-02-27T21:45:33.668Z 8 Acked 1007 Write 5 Done Done Sent false
39852024-02-27T21:45:33.668Z 9 Acked 1008 Read 10 Done Done Sent false
39862024-02-27T21:45:33.668Z 10 Acked 1009 Read 2 Done Done Sent false
39872024-02-27T21:45:33.668Z 11 Acked 1010 Read 8 Done Done Sent false
39882024-02-27T21:45:33.668Z 12 Acked 1011 Read 7 Done Done Sent false
39892024-02-27T21:45:33.668Z 13 Acked 1012 Write 8 Done Done Sent false
39902024-02-27T21:45:33.668Z 14 Acked 1013 Flush 0 Done Done Sent false
39912024-02-27T21:45:33.668Z 15 Acked 1014 Write 8 Done Done Sent false
39922024-02-27T21:45:33.668Z 16 Acked 1015 Write 9 Done Done Sent false
39932024-02-27T21:45:33.668Z 17 Acked 1016 Write 4 Done Done Sent false
39942024-02-27T21:45:33.668Z 18 Acked 1017 Write 9 Done Done Sent false
39952024-02-27T21:45:33.668Z 19 Acked 1018 Write 3 Done Done Sent false
39962024-02-27T21:45:33.668Z 20 Acked 1019 Read 6 Done Done Sent false
39972024-02-27T21:45:33.669Z 21 Acked 1020 Read 10 Done Done Sent false
39982024-02-27T21:45:33.669Z 22 Acked 1021 Write 3 Done Done Sent false
39992024-02-27T21:45:33.669Z 23 Acked 1022 Read 5 Done Done Sent false
40002024-02-27T21:45:33.669Z 24 Acked 1023 Read 6 Done Done Sent false
40012024-02-27T21:45:33.669Z 25 Acked 1024 Read 6 Done Done Sent false
40022024-02-27T21:45:33.669Z 26 Acked 1025 Write 10 Done Done Sent false
40032024-02-27T21:45:33.669Z 27 Acked 1026 Write 3 Done Done Sent false
40042024-02-27T21:45:33.669Z 28 Acked 1027 Write 6 Done Done Sent false
40052024-02-27T21:45:33.669Z 29 Acked 1028 Write 4 Done Done Sent false
40062024-02-27T21:45:33.669Z 30 Acked 1029 Read 8 Done Sent Sent false
40072024-02-27T21:45:33.669Z STATES DS:0 DS:1 DS:2 TOTAL
40082024-02-27T21:45:33.669Z New 0 0 0 0
40092024-02-27T21:45:33.669Z Sent 0 1 28 29
40102024-02-27T21:45:33.669Z Done 30 29 2 61
40112024-02-27T21:45:33.669Z Skipped 0 0 0 0
40122024-02-27T21:45:33.669Z Error 0 0 0 0
40132024-02-27T21:45:33.669ZLast Flush: 1013 1013 0
40142024-02-27T21:45:33.669ZDownstairs last five completed:
40152024-02-27T21:45:33.669ZUpstairs last five completed: 30 29 28 27 26
40162024-02-27T21:45:33.669Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:31.550590219Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1282}
40172024-02-27T21:45:33.669ZWrote out file "/tmp/test_repair_verify.data"
40182024-02-27T21:45:33.669Z
40192024-02-27T21:45:33.669ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40202024-02-27T21:45:33.669ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40212024-02-27T21:45:33.669Z 0 000-019 36 36 31 63 63 56 F F F <---
40222024-02-27T21:45:33.670Z 1 020-039 36 36 31 63 63 56 F F F <---
40232024-02-27T21:45:33.670Z 2 040-059 31 31 31 56 56 56 T T F <---
40242024-02-27T21:45:33.670Z 3 060-079 26 26 26 51 51 51 T T F <---
40252024-02-27T21:45:33.670Z 4 080-099 28 28 28 54 54 54 T T F <---
40262024-02-27T21:45:33.670Z 5 100-119 35 35 35 62 62 62 T T F <---
40272024-02-27T21:45:33.670Z 6 120-139 36 36 34 63 63 61 F F T <---
40282024-02-27T21:45:33.670Z 7 140-159 36 36 35 63 63 62 F F T <---
40292024-02-27T21:45:33.670Z 8 160-179 31 31 31 56 56 56 F F F
40302024-02-27T21:45:33.670Z 9 180-199 28 28 28 54 54 54 T T F <---
40312024-02-27T21:45:33.670Z 10 200-219 35 35 35 62 62 62 T T F <---
40322024-02-27T21:45:33.670Z 11 220-239 35 35 35 62 62 62 T T F <---
40332024-02-27T21:45:33.670Z 12 240-259 34 34 34 61 61 61 T T F <---
40342024-02-27T21:45:33.670Z 13 260-279 34 34 34 61 61 61 F F F
40352024-02-27T21:45:33.670Z 14 280-299 35 35 35 62 62 62 F F F
40362024-02-27T21:45:33.670Z 15 300-319 33 33 33 60 60 60 F F F
40372024-02-27T21:45:33.670Z 16 320-339 33 33 33 60 60 60 F F F
40382024-02-27T21:45:33.670Z 17 340-359 36 36 34 64 64 61 F F F <---
40392024-02-27T21:45:33.670Z 18 360-379 35 35 35 62 62 62 T T F <---
40402024-02-27T21:45:33.670Z 19 380-399 35 35 35 62 62 62 F F F
40412024-02-27T21:45:33.670Z 20 400-419 35 35 35 62 62 62 F F F
40422024-02-27T21:45:33.670Z 21 420-439 35 35 35 62 62 62 F F F
40432024-02-27T21:45:33.670Z 22 440-459 35 35 35 62 62 62 F F F
40442024-02-27T21:45:33.670Z 23 460-479 36 36 34 63 63 61 F F T <---
40452024-02-27T21:45:33.670Z 24 480-499 34 34 34 61 61 61 F F F
40462024-02-27T21:45:33.670Z 25 500-519 31 31 31 56 56 56 T T F <---
40472024-02-27T21:45:33.670Z 26 520-539 31 31 31 56 56 56 F F F
40482024-02-27T21:45:33.671Z 27 540-559 36 36 35 64 64 62 T T F <---
40492024-02-27T21:45:33.671Z 28 560-579 28 28 28 54 54 54 F F F
40502024-02-27T21:45:33.671Z 29 580-599 28 28 28 54 54 54 F F F
40512024-02-27T21:45:33.671ZMax gen: 36, Max flush: 64
40522024-02-27T21:45:33.671ZError: Difference in extent metadata found!
40532024-02-27T21:45:33.671ZA Difference in extent metadata is expected here
40542024-02-27T21:45:33.671ZOn loop 18
40552024-02-27T21:45:33.671Z
40562024-02-27T21:45:33.671ZVerifying data now
40572024-02-27T21:45:38.986ZNo change after verify
40582024-02-27T21:45:38.986ZLoop: 18 Downstairs dump after verify (and repair):
40592024-02-27T21:45:39.007ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40602024-02-27T21:45:39.007Z 0 000-019 36 36 36 63 63 63 F F F
40612024-02-27T21:45:39.007Z 1 020-039 36 36 36 63 63 63 F F F
40622024-02-27T21:45:39.007Z 2 040-059 37 37 37 65 65 65 F F F
40632024-02-27T21:45:39.007Z 3 060-079 37 37 37 65 65 65 F F F
40642024-02-27T21:45:39.007Z 4 080-099 37 37 37 65 65 65 F F F
40652024-02-27T21:45:39.014Z 5 100-119 37 37 37 65 65 65 F F F
40662024-02-27T21:45:39.014Z 6 120-139 36 36 36 63 63 63 F F F
40672024-02-27T21:45:39.014Z 7 140-159 36 36 36 63 63 63 F F F
40682024-02-27T21:45:39.014Z 8 160-179 31 31 31 56 56 56 F F F
40692024-02-27T21:45:39.014Z 9 180-199 37 37 37 65 65 65 F F F
40702024-02-27T21:45:39.014Z 10 200-219 37 37 37 65 65 65 F F F
40712024-02-27T21:45:39.014Z 11 220-239 37 37 37 65 65 65 F F F
40722024-02-27T21:45:39.014Z 12 240-259 37 37 37 65 65 65 F F F
40732024-02-27T21:45:39.014Z 13 260-279 34 34 34 61 61 61 F F F
40742024-02-27T21:45:39.014Z 14 280-299 35 35 35 62 62 62 F F F
40752024-02-27T21:45:39.015Z 15 300-319 33 33 33 60 60 60 F F F
40762024-02-27T21:45:39.015Z 16 320-339 33 33 33 60 60 60 F F F
40772024-02-27T21:45:39.015Z 17 340-359 36 36 36 64 64 64 F F F
40782024-02-27T21:45:39.015Z 18 360-379 37 37 37 65 65 65 F F F
40792024-02-27T21:45:39.015Z 19 380-399 35 35 35 62 62 62 F F F
40802024-02-27T21:45:39.015Z 20 400-419 35 35 35 62 62 62 F F F
40812024-02-27T21:45:39.015Z 21 420-439 35 35 35 62 62 62 F F F
40822024-02-27T21:45:39.015Z 22 440-459 35 35 35 62 62 62 F F F
40832024-02-27T21:45:39.015Z 23 460-479 36 36 36 63 63 63 F F F
40842024-02-27T21:45:39.015Z 24 480-499 34 34 34 61 61 61 F F F
40852024-02-27T21:45:39.015Z 25 500-519 37 37 37 65 65 65 F F F
40862024-02-27T21:45:39.015Z 26 520-539 31 31 31 56 56 56 F F F
40872024-02-27T21:45:39.015Z 27 540-559 37 37 37 65 65 65 F F F
40882024-02-27T21:45:39.015Z 28 560-579 28 28 28 54 54 54 F F F
40892024-02-27T21:45:39.015Z 29 580-599 28 28 28 54 54 54 F F F
40902024-02-27T21:45:39.015ZMax gen: 37, Max flush: 65
40912024-02-27T21:45:39.016Z
40922024-02-27T21:45:39.016ZBegin loop 19
40932024-02-27T21:45:39.016ZDownstairs to restart: 2
40942024-02-27T21:45:39.024Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.42660074Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290}
40952024-02-27T21:45:39.024Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.426896915Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290}
40962024-02-27T21:45:39.032Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
40972024-02-27T21:45:39.032Z,"time":"2024-02-27T21:45:37.42694145Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290}
40982024-02-27T21:45:39.033Z{"msg":"Crucible 5c475dfd-4ea1-4ea5-a67b-769f77742e29 has session id: ca88424f-0365-4201-9c75-7509ee04eda0","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.426976056Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
40992024-02-27T21:45:39.033Z{"msg":"Upstairs opts: Upstairs UUID: 5c475dfd-4ea1-4ea5-a67b-769f77742e29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427083723Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41002024-02-27T21:45:39.033Z{"msg":"Crucible stats registered with UUID: 5c475dfd-4ea1-4ea5-a67b-769f77742e29","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427116329Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41012024-02-27T21:45:39.033Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427131488Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290}
41022024-02-27T21:45:39.033Z{"msg":"5c475dfd-4ea1-4ea5-a67b-769f77742e29 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427151236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41032024-02-27T21:45:39.033Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427164924Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41042024-02-27T21:45:39.033Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427191501Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41052024-02-27T21:45:39.033Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427207169Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41062024-02-27T21:45:39.033Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427222177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41072024-02-27T21:45:39.033Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427248744Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41082024-02-27T21:45:39.033Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427272931Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41092024-02-27T21:45:39.033Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427305287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"io task","client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41102024-02-27T21:45:39.034Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427325775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"io task","client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41112024-02-27T21:45:39.034Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427348752Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"io task","client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41122024-02-27T21:45:39.034Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427557628Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"io task","client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41132024-02-27T21:45:39.034Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427685653Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"io task","client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41142024-02-27T21:45:39.034Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.427725328Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"io task","client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41152024-02-27T21:45:39.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.428245667Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41162024-02-27T21:45:39.034Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.428285302Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41172024-02-27T21:45:39.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.428312279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41182024-02-27T21:45:39.034Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.428344125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41192024-02-27T21:45:39.034Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.42880837Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41202024-02-27T21:45:39.034Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.428943284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41212024-02-27T21:45:39.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.429157539Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41222024-02-27T21:45:39.035Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.429216682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41232024-02-27T21:45:39.035Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.429293683Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41242024-02-27T21:45:39.035Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.429331779Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41252024-02-27T21:45:39.035Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.429347257Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41262024-02-27T21:45:39.035Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.429442626Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41272024-02-27T21:45:39.035Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.429479101Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41282024-02-27T21:45:39.035Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.429493929Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41292024-02-27T21:45:39.035Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.429689757Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41302024-02-27T21:45:39.035Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430120986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41312024-02-27T21:45:39.035Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430152322Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41322024-02-27T21:45:39.035Z{"msg":"[0]R flush_numbers[0..12]: [63, 63, 65, 65, 65, 65, 63, 63, 56, 65, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.43016855Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41332024-02-27T21:45:39.035Z{"msg":"[0]R generation[0..12]: [36, 36, 37, 37, 37, 37, 36, 36, 31, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430183038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41342024-02-27T21:45:39.035Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430198786Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41352024-02-27T21:45:39.035Z{"msg":"[1]R flush_numbers[0..12]: [63, 63, 65, 65, 65, 65, 63, 63, 56, 65, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430220194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41362024-02-27T21:45:39.036Z{"msg":"[1]R generation[0..12]: [36, 36, 37, 37, 37, 37, 36, 36, 31, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430234932Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41372024-02-27T21:45:39.036Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.43025644Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_idWait for a query_work_queue command to finish before sending IO
41382024-02-27T21:45:39.036Z":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41392024-02-27T21:45:39.036Z{"msg":"[2]R flush_numbers[0..12]: [63, 63, 65, 65, 65, 65, 63, 63, 56, 65, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430275057Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41402024-02-27T21:45:39.036Z{"msg":"[2]R generation[0..12]: [36, 36, 37, 37, 37, 37, 36, 36, 31, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430295845Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41412024-02-27T21:45:39.036Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430310373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41422024-02-27T21:45:39.036Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430330161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41432024-02-27T21:45:39.036Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430344119Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41442024-02-27T21:45:39.036Z{"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430369086Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41452024-02-27T21:45:39.036Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430383925Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41462024-02-27T21:45:39.036Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430416761Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41472024-02-27T21:45:39.036Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430429899Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41482024-02-27T21:45:39.036Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430443837Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41492024-02-27T21:45:39.036Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430464005Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41502024-02-27T21:45:39.036Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430477784Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41512024-02-27T21:45:39.037Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430496581Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41522024-02-27T21:45:39.037Z{"msg":"5c475dfd-4ea1-4ea5-a67b-769f77742e29 is now active with session: ca88424f-0365-4201-9c75-7509ee04eda0","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.43050985Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41532024-02-27T21:45:39.037Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430528348Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290,"session_id":"ca88424f-0365-4201-9c75-7509ee04eda0"}
41542024-02-27T21:45:39.037Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:37.430541246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290}
41552024-02-27T21:45:39.037ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41562024-02-27T21:45:39.037ZLoading write count information from file "/tmp/test_repair_verify.data"
41572024-02-27T21:45:39.037ZRun Repair workload
41582024-02-27T21:45:39.037Z01/30 Write block 174 len 16384 data: 5 5 4 4
41592024-02-27T21:45:39.037Z02/30 Write block 304 len 36864 data: 5 4 5 7 7 8 8 8 7
41602024-02-27T21:45:39.037Z03/30 Read block 399 len 8192
41612024-02-27T21:45:39.055Z04/30 Write block 200 len 20480 data: 4 4 4 4 4
41622024-02-27T21:45:39.056Z05/30 Write block 181 len 16384 data: 3 2 2 3
41632024-02-27T21:45:39.056Z06/30 Write block 221 len 28672 data: 5 7 6 7 5 6 6
41642024-02-27T21:45:39.056Z07/30 Write block 117 len 8192 data: 4 4
41652024-02-27T21:45:39.056Z08/30 Write block 312 len 36864 data: 8 6 6 4 6 4 4 5 5
41662024-02-27T21:45:39.056Z09/30 Read block 6 len 12288
41672024-02-27T21:45:39.056Z10/30 Write block 433 len 8192 data: 6 7
41682024-02-27T21:45:39.056Z11/30 Write block 526 len 24576 data: 2 3 3 3 3 2
41692024-02-27T21:45:39.057Z12/30 Write block 20 len 28672 data: 4 4 4 4 5 5 4
41702024-02-27T21:45:39.057Z13/30 Read block 90 len 40960
41712024-02-27T21:45:39.057Z14/30 Read block 125 len 32768
41722024-02-27T21:45:39.057Z15/30 Write block 304 len 40960 data: 6 5 6 8 8 9 9 9 9 7
41732024-02-27T21:45:39.057Z16/30 Flush
41742024-02-27T21:45:39.066Z17/30 Write block 509 len 16384 data: 4 5 6 6
41752024-02-27T21:45:39.066Z18/30 Flush
41762024-02-27T21:45:39.066Z19/30 Write block 586 len 8192 data: 4 4
41772024-02-27T21:45:39.067Z20/30 Read block 105 len 32768
41782024-02-27T21:45:39.067Z21/30 Write block 345 len 12288 data: 5 5 5
41792024-02-27T21:45:39.067Z22/30 Read block 185 len 12288
41802024-02-27T21:45:39.067Z23/30 Write block 182 len 24576 data: 3 3 4 3 5 5
41812024-02-27T21:45:39.067Z24/30 Write block 12 len 20480 data: 5 4 6 5 5
41822024-02-27T21:45:39.067Z25/30 Read block 488 len 40960
41832024-02-27T21:45:39.067Z26/30 Write block 596 len 16384 data: 3 3 3 2
41842024-02-27T21:45:39.068Z27/30 Write block 24 len 24576 data: 6 6 5 4 4 5
41852024-02-27T21:45:39.068Z28/30 Write block 98 len 36864 data: 4 4 4 4 4 4 4 5 6
41862024-02-27T21:45:39.068Z29/30 Write block 587 len 4096 data: 5
41872024-02-27T21:45:39.068Z30/30 Read block 116 len 4096
41882024-02-27T21:45:39.068Z----------------------------------------------------------------
41892024-02-27T21:45:39.068Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
41902024-02-27T21:45:39.069ZStates: Active Active Active
41912024-02-27T21:45:39.069ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
41922024-02-27T21:45:39.069Z 1 Acked 1000 Write 4 Done Done Done false
41932024-02-27T21:45:39.069Z 2 Acked 1001 Write 9 Done Done Done false
41942024-02-27T21:45:39.069Z 3 Acked 1002 Read 2 Done Done Done false
41952024-02-27T21:45:39.069Z 4 Acked 1003 Write 5 Done Done Sent false
41962024-02-27T21:45:39.070Z 5 Acked 1004 Write 4 Done Done Sent false
41972024-02-27T21:45:39.070Z 6 Acked 1005 Write 7 Done Done Sent false
41982024-02-27T21:45:39.070Z 7 Acked 1006 Write 2 Done Done Sent false
41992024-02-27T21:45:39.070Z 8 Acked 1007 Write 9 Done Done Sent false
42002024-02-27T21:45:39.070Z 9 Acked 1008 Read 3 Done Done Sent false
42012024-02-27T21:45:39.070Z 10 Acked 1009 Write 2 Done Done Sent false
42022024-02-27T21:45:39.070Z 11 Acked 1010 Write 6 Done Done Sent false
42032024-02-27T21:45:39.070Z 12 Acked 1011 Write 7 Done Done Sent false
42042024-02-27T21:45:39.071Z 13 Acked 1012 Read 10 Done Done Sent false
42052024-02-27T21:45:39.071Z 14 Acked 1013 Read 8 Done Done Sent false
42062024-02-27T21:45:39.071Z 15 Acked 1014 Write 10 Done Done Sent false
42072024-02-27T21:45:39.071Z 16 Acked 1015 Flush 0 Done Done Sent false
42082024-02-27T21:45:39.071Z 17 Acked 1016 Write 4 Done Done Sent false
42092024-02-27T21:45:39.071Z 18 Acked 1017 Flush 0 Done Done Sent false
42102024-02-27T21:45:39.072Z 19 Acked 1018 Write 2 Done Done Sent false
42112024-02-27T21:45:39.072Z 20 Acked 1019 Read 8 Done Done Sent false
42122024-02-27T21:45:39.072Z 21 Acked 1020 Write 3 Done Done Sent false
42132024-02-27T21:45:39.072Z 22 Acked 1021 Read 3 Done Done Sent false
42142024-02-27T21:45:39.072Z 23 Acked 1022 Write 6 Done Done Sent false
42152024-02-27T21:45:39.072Z 24 Acked 1023 Write 5 Done Done Sent false
42162024-02-27T21:45:39.072Z 25 Acked 1024 Read 10 Done Done Sent false
42172024-02-27T21:45:39.073Z 26 Acked 1025 Write 4 Done Done Sent false
42182024-02-27T21:45:39.073Z 27 Acked 1026 Write 6 Done Done Sent false
42192024-02-27T21:45:39.073Z 28 Acked 1027 Write 9 Done Done Sent false
42202024-02-27T21:45:39.073Z 29 Acked 1028 Write 1 Done Done Sent false
42212024-02-27T21:45:39.073Z 30 Acked 1029 Read 1 Sent Done Sent false
42222024-02-27T21:45:39.073Z STATES DS:0 DS:1 DS:2 TOTAL
42232024-02-27T21:45:39.073Z New 0 0 0 0
42242024-02-27T21:45:39.073Z Sent 1 0 27 28
42252024-02-27T21:45:39.073Z Done 29 30 3 62
42262024-02-27T21:45:39.073Z Skipped 0 0 0 0
42272024-02-27T21:45:39.074Z Error 0 0 0 0
42282024-02-27T21:45:39.074ZLast Flush: 1017 1017 0
42292024-02-27T21:45:39.074ZDownstairs last five completed:
42302024-02-27T21:45:39.074ZUpstairs last five completed: 30 29 28 27 26
42312024-02-27T21:45:39.074Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:37.46158129Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1290}
42322024-02-27T21:45:39.074ZWrote out file "/tmp/test_repair_verify.data"
42332024-02-27T21:45:39.074Z
42342024-02-27T21:45:39.083ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42352024-02-27T21:45:39.107ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42362024-02-27T21:45:39.107Z 0 000-019 36 36 36 63 63 63 T T F <---
42372024-02-27T21:45:39.107Z 1 020-039 38 38 36 66 66 63 T T F <---
42382024-02-27T21:45:39.107Z 2 040-059 37 37 37 65 65 65 F F F
42392024-02-27T21:45:39.108Z 3 060-079 37 37 37 65 65 65 F F F
42402024-02-27T21:45:39.108Z 4 080-099 37 37 37 65 65 65 T T F <---
42412024-02-27T21:45:39.108Z 5 100-119 38 38 37 66 66 65 T T F <---
42422024-02-27T21:45:39.111Z 6 120-139 36 36 36 63 63 63 F F F
42432024-02-27T21:45:39.112Z 7 140-159 36 36 36 63 63 63 F F F
42442024-02-27T21:45:39.112Z 8 160-179 38 38 31 66 66 56 F F T <---
42452024-02-27T21:45:39.112Z 9 180-199 38 38 37 66 66 65 T T F <---
42462024-02-27T21:45:39.112Z 10 200-219 38 38 37 66 66 65 F F F <---
42472024-02-27T21:45:39.113Z 11 220-239 38 38 37 66 66 65 F F F <---
42482024-02-27T21:45:39.113Z 12 240-259 37 37 37 65 65 65 F F F
42492024-02-27T21:45:39.113Z 13 260-279 34 34 34 61 61 61 F F F
42502024-02-27T21:45:39.114Z 14 280-299 35 35 35 62 62 62 F F F
42512024-02-27T21:45:39.114Z 15 300-319 38 38 33 66 66 60 F F T <---
42522024-02-27T21:45:39.114Z 16 320-339 38 38 33 66 66 60 F F F <---
42532024-02-27T21:45:39.115Z 17 340-359 36 36 36 64 64 64 T T F <---
42542024-02-27T21:45:39.116Z 18 360-379 37 37 37 65 65 65 F F F
42552024-02-27T21:45:39.117Z 19 380-399 35 35 35 62 62 62 F F F
42562024-02-27T21:45:39.117Z 20 400-419 35 35 35 62 62 62 F F F
42572024-02-27T21:45:39.119Z 21 420-439 38 38 35 66 66 62 F F F <---
42582024-02-27T21:45:39.119Z 22 440-459 35 35 35 62 62 62 F F F
42592024-02-27T21:45:39.119Z 23 460-479 36 36 36 63 63 63 F F F
42602024-02-27T21:45:39.119Z 24 480-499 34 34 34 61 61 61 F F F
42612024-02-27T21:45:39.119Z 25 500-519 38 38 37 67 67 65 F F F <---
42622024-02-27T21:45:39.119Z 26 520-539 38 38 31 66 66 56 F F F <---
42632024-02-27T21:45:39.120Z 27 540-559 37 37 37 65 65 65 F F F
42642024-02-27T21:45:39.120Z 28 560-579 28 28 28 54 54 54 F F F
42652024-02-27T21:45:39.120Z 29 580-599 28 28 28 54 54 54 T T F <---
42662024-02-27T21:45:39.120ZMax gen: 38, Max flush: 67
42672024-02-27T21:45:39.120ZError: Difference in extent metadata found!
42682024-02-27T21:45:39.122ZA Difference in extent metadata is expected here
42692024-02-27T21:45:39.123ZOn loop 19
42702024-02-27T21:45:39.123Z
42712024-02-27T21:45:39.123ZVerifying data now
42722024-02-27T21:45:44.825ZNo change after verify
42732024-02-27T21:45:44.825ZLoop: 19 Downstairs dump after verify (and repair):
42742024-02-27T21:45:44.840ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42752024-02-27T21:45:44.840Z 0 000-019 39 39 39 68 68 68 F F F
42762024-02-27T21:45:44.840Z 1 020-039 39 39 39 68 68 68 F F F
42772024-02-27T21:45:44.840Z 2 040-059 37 37 37 65 65 65 F F F
42782024-02-27T21:45:44.840Z 3 060-079 37 37 37 65 65 65 F F F
42792024-02-27T21:45:44.840Z 4 080-099 39 39 39 68 68 68 F F F
42802024-02-27T21:45:44.840Z 5 100-119 39 39 39 68 68 68 F F F
42812024-02-27T21:45:44.840Z 6 120-139 36 36 36 63 63 63 F F F
42822024-02-27T21:45:44.840Z 7 140-159 36 36 36 63 63 63 F F F
42832024-02-27T21:45:44.840Z 8 160-179 38 38 38 66 66 66 F F F
42842024-02-27T21:45:44.840Z 9 180-199 39 39 39 68 68 68 F F F
42852024-02-27T21:45:44.846Z 10 200-219 38 38 38 66 66 66 F F F
42862024-02-27T21:45:44.846Z 11 220-239 38 38 38 66 66 66 F F F
42872024-02-27T21:45:44.846Z 12 240-259 37 37 37 65 65 65 F F F
42882024-02-27T21:45:44.846Z 13 260-279 34 34 34 61 61 61 F F F
42892024-02-27T21:45:44.846Z 14 280-299 35 35 35 62 62 62 F F F
42902024-02-27T21:45:44.846Z 15 300-319 38 38 38 66 66 66 F F F
42912024-02-27T21:45:44.846Z 16 320-339 38 38 38 66 66 66 F F F
42922024-02-27T21:45:44.846Z 17 340-359 39 39 39 68 68 68 F F F
42932024-02-27T21:45:44.846Z 18 360-379 37 37 37 65 65 65 F F F
42942024-02-27T21:45:44.846Z 19 380-399 35 35 35 62 62 62 F F F
42952024-02-27T21:45:44.846Z 20 400-419 35 35 35 62 62 62 F F F
42962024-02-27T21:45:44.846Z 21 420-439 38 38 38 66 66 66 F F F
42972024-02-27T21:45:44.846Z 22 440-459 35 35 35 62 62 62 F F F
42982024-02-27T21:45:44.846Z 23 460-479 36 36 36 63 63 63 F F F
42992024-02-27T21:45:44.846Z 24 480-499 34 34 34 61 61 61 F F F
43002024-02-27T21:45:44.846Z 25 500-519 38 38 38 67 67 67 F F F
43012024-02-27T21:45:44.846Z 26 520-539 38 38 38 66 66 66 F F F
43022024-02-27T21:45:44.846Z 27 540-559 37 37 37 65 65 65 F F F
43032024-02-27T21:45:44.846Z 28 560-579 28 28 28 54 54 54 F F F
43042024-02-27T21:45:44.846Z 29 580-599 39 39 39 68 68 68 F F F
43052024-02-27T21:45:44.846ZMax gen: 39, Max flush: 68
43062024-02-27T21:45:44.847Z
43072024-02-27T21:45:44.847ZBegin loop 20
43082024-02-27T21:45:44.847ZDownstairs to restart: 0
43092024-02-27T21:45:44.865Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.268858017Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298}
43102024-02-27T21:45:44.865Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",Crucible runtime is spawned
43112024-02-27T21:45:44.873Z\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269230833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298}
43122024-02-27T21:45:44.873Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269289236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298}
43132024-02-27T21:45:44.874Z{"msg":"Crucible dd51b1bf-aa62-47dc-8797-a77d9d5d8b50 has session id: b26fb833-ee44-4237-ba81-0c913b1f8978","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269316053Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43142024-02-27T21:45:44.874Z{"msg":"Upstairs opts: Upstairs UUID: dd51b1bf-aa62-47dc-8797-a77d9d5d8b50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269367237Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43152024-02-27T21:45:44.874Z{"msg":"Crucible stats registered with UUID: dd51b1bf-aa62-47dc-8797-a77d9d5d8b50","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269408612Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43162024-02-27T21:45:44.874Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269449427Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298}
43172024-02-27T21:45:44.874Z{"msg":"dd51b1bf-aa62-47dc-8797-a77d9d5d8b50 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269484323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43182024-02-27T21:45:44.874Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269529178Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43192024-02-27T21:45:44.874Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269574343Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43202024-02-27T21:45:44.874Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269616688Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43212024-02-27T21:45:44.874Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269657473Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43222024-02-27T21:45:44.874Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.26967969Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43232024-02-27T21:45:44.874Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269702657Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43242024-02-27T21:45:44.874Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269744703Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"io task","client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43252024-02-27T21:45:44.874Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269780088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"io task","client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43262024-02-27T21:45:44.874Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269825773Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"io task","client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43272024-02-27T21:45:44.874Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269869998Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"io task","client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43282024-02-27T21:45:44.874Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269899245Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"io task","client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43292024-02-27T21:45:44.874Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.269927541Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"io task","client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43302024-02-27T21:45:44.874Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.270584814Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43312024-02-27T21:45:44.874Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.270624739Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43322024-02-27T21:45:44.874Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.270653276Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43332024-02-27T21:45:44.874Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.270679033Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43342024-02-27T21:45:44.874Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271179604Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43352024-02-27T21:45:44.874Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271318207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43362024-02-27T21:45:44.874Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271364202Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43372024-02-27T21:45:44.874Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271404647Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43382024-02-27T21:45:44.875Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.27163531Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43392024-02-27T21:45:44.875Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271681324Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43402024-02-27T21:45:44.875Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271707361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43412024-02-27T21:45:44.875Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271914627Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43422024-02-27T21:45:44.875Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271952233Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43432024-02-27T21:45:44.875Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271968671Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43442024-02-27T21:45:44.875Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.271990748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43452024-02-27T21:45:44.875Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.27240108Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43462024-02-27T21:45:44.875Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272438255Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43472024-02-27T21:45:44.875Z{"msg":"[0]R flush_numbers[0..12]: [68, 68, 65, 65, 68, 68, 63, 63, 66, 68, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272454953Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43482024-02-27T21:45:44.875Z{"msg":"[0]R generation[0..12]: [39, 39, 37, 37, 39, 39, 36, 36, 38, 39, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.27248419Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43492024-02-27T21:45:44.875Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272509447Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43502024-02-27T21:45:44.875Z{"msg":"[1]R flush_numbers[0..12]: [68, 68, 65, 65, 68, 68, 63, 63, 66, 68, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272537024Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43512024-02-27T21:45:44.875Z{"msg":"[1]R generation[0..12]: [39, 39, 37, 37, 39, 39, 36, 36, 38, 39, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272551902Z","hostname":"Wait for a query_work_queue command to finish before sending IO
43522024-02-27T21:45:44.875Zip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43532024-02-27T21:45:44.875Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272577629Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43542024-02-27T21:45:44.875Z{"msg":"[2]R flush_numbers[0..12]: [68, 68, 65, 65, 68, 68, 63, 63, 66, 68, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272592917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43552024-02-27T21:45:44.875Z{"msg":"[2]R generation[0..12]: [39, 39, 37, 37, 39, 39, 36, 36, 38, 39, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272613565Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43562024-02-27T21:45:44.875Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272629123Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43572024-02-27T21:45:44.875Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272642451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43582024-02-27T21:45:44.875Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272663819Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43592024-02-27T21:45:44.875Z{"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272677217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43602024-02-27T21:45:44.875Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272697295Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43612024-02-27T21:45:44.876Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272711043Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43622024-02-27T21:45:44.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272730461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43632024-02-27T21:45:44.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272758738Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43642024-02-27T21:45:44.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272779235Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43652024-02-27T21:45:44.876Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272815621Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43662024-02-27T21:45:44.876Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272847917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43672024-02-27T21:45:44.876Z{"msg":"dd51b1bf-aa62-47dc-8797-a77d9d5d8b50 is now active with session: b26fb833-ee44-4237-ba81-0c913b1f8978","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272863065Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43682024-02-27T21:45:44.876Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272888312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298,"session_id":"b26fb833-ee44-4237-ba81-0c913b1f8978"}
43692024-02-27T21:45:44.876Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:43.272913349Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298}
43702024-02-27T21:45:44.876ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43712024-02-27T21:45:44.876ZLoading write count information from file "/tmp/test_repair_verify.data"
43722024-02-27T21:45:44.876ZRun Repair workload
43732024-02-27T21:45:44.879Z01/30 Write block 121 len 36864 data: 5 5 5 4 4 4 4 4 3
43742024-02-27T21:45:44.880Z02/30 Write block 289 len 40960 data: 6 5 4 4 4 4 4 3 3 3
43752024-02-27T21:45:44.880Z03/30 Read block 354 len 12288
43762024-02-27T21:45:44.880Z04/30 Read block 158 len 8192
43772024-02-27T21:45:44.880Z05/30 Flush
43782024-02-27T21:45:44.880Z06/30 Write block 397 len 40960 data: 4 3 3 4 5 4 3 3 2 3
43792024-02-27T21:45:44.880Z07/30 Read block 310 len 36864
43802024-02-27T21:45:44.880Z08/30 Flush
43812024-02-27T21:45:44.883Z09/30 Write block 271 len 16384 data: 4 3 3 3
43822024-02-27T21:45:44.883Z10/30 Read block 272 len 28672
43832024-02-27T21:45:44.883Z11/30 Read block 571 len 32768
43842024-02-27T21:45:44.883Z12/30 Read block 76 len 20480
43852024-02-27T21:45:44.886Z13/30 Write block 488 len 36864 data: 4 5 5 6 7 6 6 4 4
43862024-02-27T21:45:44.886Z14/30 Read block 524 len 24576
43872024-02-27T21:45:44.886Z15/30 Read block 98 len 36864
43882024-02-27T21:45:44.889Z16/30 Read block 100 len 28672
43892024-02-27T21:45:44.889Z17/30 Read block 483 len 28672
43902024-02-27T21:45:44.889Z18/30 Read block 485 len 8192
43912024-02-27T21:45:44.892Z19/30 Flush
43922024-02-27T21:45:44.892Z20/30 Write block 151 len 8192 data: 5 5
43932024-02-27T21:45:44.895Z21/30 Write block 128 len 16384 data: 5 4 3 3
43942024-02-27T21:45:44.896Z22/30 Read block 125 len 8192
43952024-02-27T21:45:44.896Z23/30 Read block 574 len 12288
43962024-02-27T21:45:44.896Z24/30 Read block 485 len 40960
43972024-02-27T21:45:44.899Z25/30 Read block 107 len 16384
43982024-02-27T21:45:44.899Z26/30 Write block 434 len 8192 data: 8 5
43992024-02-27T21:45:44.899Z27/30 Read block 445 len 32768
44002024-02-27T21:45:44.899Z28/30 Write block 277 len 8192 data: 4 4
44012024-02-27T21:45:44.907Z29/30 Write block 99 len 8192 data: 5 5
44022024-02-27T21:45:44.907Z30/30 Write block 571 len 12288 data: 4 4 5
44032024-02-27T21:45:44.907Z----------------------------------------------------------------
44042024-02-27T21:45:44.907Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44052024-02-27T21:45:44.907ZStates: Active Active Active
44062024-02-27T21:45:44.908ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44072024-02-27T21:45:44.908Z 1 Acked 1000 Write 9 Done Done Done false
44082024-02-27T21:45:44.908Z 2 Acked 1001 Write 10 Sent Done Done false
44092024-02-27T21:45:44.908Z 3 Acked 1002 Read 3 Sent Done Done false
44102024-02-27T21:45:44.908Z 4 Acked 1003 Read 2 Sent Done Done false
44112024-02-27T21:45:44.908Z 5 Acked 1004 Flush 0 Sent Done Done false
44122024-02-27T21:45:44.908Z 6 Acked 1005 Write 10 Sent Done Done false
44132024-02-27T21:45:44.908Z 7 Acked 1006 Read 9 Sent Done Done false
44142024-02-27T21:45:44.908Z 8 Acked 1007 Flush 0 Sent Done Done false
44152024-02-27T21:45:44.908Z 9 Acked 1008 Write 4 Sent Done Done false
44162024-02-27T21:45:44.908Z 10 Acked 1009 Read 7 Sent Done Done false
44172024-02-27T21:45:44.908Z 11 Acked 1010 Read 8 Sent Done Done false
44182024-02-27T21:45:44.908Z 12 Acked 1011 Read 5 Sent Done Done false
44192024-02-27T21:45:44.908Z 13 Acked 1012 Write 9 Sent Done Done false
44202024-02-27T21:45:44.908Z 14 Acked 1013 Read 6 Sent Done Done false
44212024-02-27T21:45:44.908Z 15 Acked 1014 Read 9 Sent Done Done false
44222024-02-27T21:45:44.908Z 16 Acked 1015 Read 7 Sent Done Done false
44232024-02-27T21:45:44.908Z 17 Acked 1016 Read 7 Sent Done Done false
44242024-02-27T21:45:44.908Z 18 Acked 1017 Read 2 Sent Done Done false
44252024-02-27T21:45:44.908Z 19 Acked 1018 Flush 0 Sent Done Done false
44262024-02-27T21:45:44.908Z 20 Acked 1019 Write 2 Sent Done Done false
44272024-02-27T21:45:44.908Z 21 Acked 1020 Write 4 Sent Done Done false
44282024-02-27T21:45:44.908Z 22 Acked 1021 Read 2 Sent Done Done false
44292024-02-27T21:45:44.908Z 23 Acked 1022 Read 3 Sent Done Done false
44302024-02-27T21:45:44.908Z 24 Acked 1023 Read 10 Sent Done Done false
44312024-02-27T21:45:44.909Z 25 Acked 1024 Read 4 Sent Done Done false
44322024-02-27T21:45:44.909Z 26 Acked 1025 Write 2 Sent Done Done false
44332024-02-27T21:45:44.909Z 27 Acked 1026 Read 8 Sent Done Done false
44342024-02-27T21:45:44.909Z 28 Acked 1027 Write 2 Sent Done Done false
44352024-02-27T21:45:44.909Z 29 Acked 1028 Write 2 Sent Done Done false
44362024-02-27T21:45:44.909Z 30 Acked 1029 Write 3 Sent Sent Sent false
44372024-02-27T21:45:44.909Z STATES DS:0 DS:1 DS:2 TOTAL
44382024-02-27T21:45:44.909Z New 0 0 0 0
44392024-02-27T21:45:44.909Z Sent 29 1 1 31
44402024-02-27T21:45:44.909Z Done 1 29 29 59
44412024-02-27T21:45:44.909Z Skipped 0 0 0 0
44422024-02-27T21:45:44.909Z Error 0 0 0 0
44432024-02-27T21:45:44.909ZLast Flush: 0 1018 1018
44442024-02-27T21:45:44.909ZDownstairs last five completed:
44452024-02-27T21:45:44.909ZUpstairs last five completed: 30 29 28 27 26
44462024-02-27T21:45:44.909Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:43.305454216Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1298}
44472024-02-27T21:45:44.909ZWrote out file "/tmp/test_repair_verify.data"
44482024-02-27T21:45:44.909Z
44492024-02-27T21:45:44.913ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44502024-02-27T21:45:44.930ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44512024-02-27T21:45:44.930Z 0 000-019 39 39 39 68 68 68 F F F
44522024-02-27T21:45:44.930Z 1 020-039 39 39 39 68 68 68 F F F
44532024-02-27T21:45:44.930Z 2 040-059 37 37 37 65 65 65 F F F
44542024-02-27T21:45:44.930Z 3 060-079 37 37 37 65 65 65 F F F
44552024-02-27T21:45:44.930Z 4 080-099 39 39 39 68 68 68 F T T <---
44562024-02-27T21:45:44.930Z 5 100-119 39 39 39 68 68 68 F T T <---
44572024-02-27T21:45:44.930Z 6 120-139 36 40 40 63 69 69 T T T <---
44582024-02-27T21:45:44.930Z 7 140-159 36 36 36 63 63 63 F T T <---
44592024-02-27T21:45:44.930Z 8 160-179 38 38 38 66 66 66 F F F
44602024-02-27T21:45:44.934Z 9 180-199 39 39 39 68 68 68 F F F
44612024-02-27T21:45:44.934Z 10 200-219 38 38 38 66 66 66 F F F
44622024-02-27T21:45:44.934Z 11 220-239 38 38 38 66 66 66 F F F
44632024-02-27T21:45:44.934Z 12 240-259 37 37 37 65 65 65 F F F
44642024-02-27T21:45:44.934Z 13 260-279 34 40 40 61 71 71 F T T <---
44652024-02-27T21:45:44.934Z 14 280-299 35 40 40 62 69 69 F F F <---
44662024-02-27T21:45:44.934Z 15 300-319 38 38 38 66 66 66 F F F
44672024-02-27T21:45:44.934Z 16 320-339 38 38 38 66 66 66 F F F
44682024-02-27T21:45:44.934Z 17 340-359 39 39 39 68 68 68 F F F
44692024-02-27T21:45:44.934Z 18 360-379 37 37 37 65 65 65 F F F
44702024-02-27T21:45:44.934Z 19 380-399 35 40 40 62 70 70 F F F <---
44712024-02-27T21:45:44.934Z 20 400-419 35 40 40 62 70 70 F F F <---
44722024-02-27T21:45:44.934Z 21 420-439 38 38 38 66 66 66 F T T <---
44732024-02-27T21:45:44.934Z 22 440-459 35 35 35 62 62 62 F F F
44742024-02-27T21:45:44.934Z 23 460-479 36 36 36 63 63 63 F F F
44752024-02-27T21:45:44.934Z 24 480-499 34 40 40 61 71 71 F F F <---
44762024-02-27T21:45:44.934Z 25 500-519 38 38 38 67 67 67 F F F
44772024-02-27T21:45:44.934Z 26 520-539 38 38 38 66 66 66 F F F
44782024-02-27T21:45:44.934Z 27 540-559 37 37 37 65 65 65 F F F
44792024-02-27T21:45:44.934Z 28 560-579 28 28 28 54 54 54 F T T <---
44802024-02-27T21:45:44.934Z 29 580-599 39 39 39 68 68 68 F F F
44812024-02-27T21:45:44.935ZMax gen: 40, Max flush: 71
44822024-02-27T21:45:44.935ZError: Difference in extent metadata found!
44832024-02-27T21:45:44.935ZA Difference in extent metadata is expected here
44842024-02-27T21:45:44.935ZOn loop 20
44852024-02-27T21:45:44.935Z
44862024-02-27T21:45:44.935ZVerifying data now
44872024-02-27T21:45:49.498ZNo change after verify
44882024-02-27T21:45:49.499ZLoop: 20 Downstairs dump after verify (and repair):
44892024-02-27T21:45:49.517ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44902024-02-27T21:45:49.517Z 0 000-019 39 39 39 68 68 68 F F F
44912024-02-27T21:45:49.517Z 1 020-039 39 39 39 68 68 68 F F F
44922024-02-27T21:45:49.517Z 2 040-059 37 37 37 65 65 65 F F F
44932024-02-27T21:45:49.521Z 3 060-079 37 37 37 65 65 65 F F F
44942024-02-27T21:45:49.521Z 4 080-099 41 41 41 72 72 72 F F F
44952024-02-27T21:45:49.521Z 5 100-119 41 41 41 72 72 72 F F F
44962024-02-27T21:45:49.521Z 6 120-139 41 41 41 72 72 72 F F F
44972024-02-27T21:45:49.521Z 7 140-159 41 41 41 72 72 72 F F F
44982024-02-27T21:45:49.521Z 8 160-179 38 38 38 66 66 66 F F F
44992024-02-27T21:45:49.521Z 9 180-199 39 39 39 68 68 68 F F F
45002024-02-27T21:45:49.521Z 10 200-219 38 38 38 66 66 66 F F F
45012024-02-27T21:45:49.522Z 11 220-239 38 38 38 66 66 66 F F F
45022024-02-27T21:45:49.522Z 12 240-259 37 37 37 65 65 65 F F F
45032024-02-27T21:45:49.522Z 13 260-279 41 41 41 72 72 72 F F F
45042024-02-27T21:45:49.522Z 14 280-299 40 40 40 69 69 69 F F F
45052024-02-27T21:45:49.522Z 15 300-319 38 38 38 66 66 66 F F F
45062024-02-27T21:45:49.522Z 16 320-339 38 38 38 66 66 66 F F F
45072024-02-27T21:45:49.522Z 17 340-359 39 39 39 68 68 68 F F F
45082024-02-27T21:45:49.522Z 18 360-379 37 37 37 65 65 65 F F F
45092024-02-27T21:45:49.522Z 19 380-399 40 40 40 70 70 70 F F F
45102024-02-27T21:45:49.522Z 20 400-419 40 40 40 70 70 70 F F F
45112024-02-27T21:45:49.523Z 21 420-439 41 41 41 72 72 72 F F F
45122024-02-27T21:45:49.523Z 22 440-459 35 35 35 62 62 62 F F F
45132024-02-27T21:45:49.523Z 23 460-479 36 36 36 63 63 63 F F F
45142024-02-27T21:45:49.523Z 24 480-499 40 40 40 71 71 71 F F F
45152024-02-27T21:45:49.523Z 25 500-519 38 38 38 67 67 67 F F F
45162024-02-27T21:45:49.523Z 26 520-539 38 38 38 66 66 66 F F F
45172024-02-27T21:45:49.523Z 27 540-559 37 37 37 65 65 65 F F F
45182024-02-27T21:45:49.523Z 28 560-579 41 41 41 72 72 72 F F F
45192024-02-27T21:45:49.523Z 29 580-599 39 39 39 68 68 68 F F F
45202024-02-27T21:45:49.523ZMax gen: 41, Max flush: 72
45212024-02-27T21:45:49.523Z
45222024-02-27T21:45:49.523ZBegin loop 21
45232024-02-27T21:45:49.524ZDownstairs to restart: 0
45242024-02-27T21:45:49.535Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.935750053Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306}
45252024-02-27T21:45:49.535Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936056377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306}
45262024-02-27T21:45:49.544Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936090393Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306}
45272024-02-27T21:45:49.545Z{"msg":"Crucible 77df52bc-e899-4e8a-bd2d-0baf3f98e763 has session id: 797e0c4e-944f-45f9-8384-412f04612a48","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936119639Z","hostnameCrucible runtime is spawned
45282024-02-27T21:45:49.545Z":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45292024-02-27T21:45:49.545Z{"msg":"Upstairs opts: Upstairs UUID: 77df52bc-e899-4e8a-bd2d-0baf3f98e763, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936178912Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45302024-02-27T21:45:49.545Z{"msg":"Crucible stats registered with UUID: 77df52bc-e899-4e8a-bd2d-0baf3f98e763","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936229676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45312024-02-27T21:45:49.545Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936263702Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306}
45322024-02-27T21:45:49.546Z{"msg":"77df52bc-e899-4e8a-bd2d-0baf3f98e763 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936398036Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45332024-02-27T21:45:49.546Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936444161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45342024-02-27T21:45:49.546Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936497295Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45352024-02-27T21:45:49.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.93653939Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45362024-02-27T21:45:49.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936609982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45372024-02-27T21:45:49.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936659306Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45382024-02-27T21:45:49.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936687013Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45392024-02-27T21:45:49.547Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936711679Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"io task","client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45402024-02-27T21:45:49.548Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936743666Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"io task","client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45412024-02-27T21:45:49.548Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936769743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"io task","client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45422024-02-27T21:45:49.548Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.93679434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"io task","client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45432024-02-27T21:45:49.548Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936818237Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"io task","client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45442024-02-27T21:45:49.548Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.936842444Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"io task","client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45452024-02-27T21:45:49.548Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.937410837Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45462024-02-27T21:45:49.548Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.937452912Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45472024-02-27T21:45:49.549Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.93747596Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45482024-02-27T21:45:49.549Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.937505236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45492024-02-27T21:45:49.549Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.937547001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45502024-02-27T21:45:49.549Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.937582987Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45512024-02-27T21:45:49.549Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938016446Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45522024-02-27T21:45:49.549Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.93806965Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45532024-02-27T21:45:49.549Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938178747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45542024-02-27T21:45:49.550Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938568411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45552024-02-27T21:45:49.550Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938623164Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45562024-02-27T21:45:49.550Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938652741Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45572024-02-27T21:45:49.550Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938683007Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45582024-02-27T21:45:49.550Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938707984Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45592024-02-27T21:45:49.551Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938736241Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45602024-02-27T21:45:49.551Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938758888Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45612024-02-27T21:45:49.551Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938786625Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45622024-02-27T21:45:49.551Z{"msg":"[0]R flush_numbers[0..12]: [68, 68, 65, 65, 72, 72, 72, 72, 66, 68, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938811942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45632024-02-27T21:45:49.551Z{"msg":"[0]R generation[0..12]: [39, 39, 37, 37, 41, 41, 41, 41, 38, 39, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938839879Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45642024-02-27T21:45:49.552Z{"msg":"Wait for a query_work_queue command to finish before sending IO
45652024-02-27T21:45:49.552Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938867326Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45662024-02-27T21:45:49.552Z{"msg":"[1]R flush_numbers[0..12]: [68, 68, 65, 65, 72, 72, 72, 72, 66, 68, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938899052Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45672024-02-27T21:45:49.552Z{"msg":"[1]R generation[0..12]: [39, 39, 37, 37, 41, 41, 41, 41, 38, 39, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938925549Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45682024-02-27T21:45:49.552Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938956175Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45692024-02-27T21:45:49.552Z{"msg":"[2]R flush_numbers[0..12]: [68, 68, 65, 65, 72, 72, 72, 72, 66, 68, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.938982442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45702024-02-27T21:45:49.552Z{"msg":"[2]R generation[0..12]: [39, 39, 37, 37, 41, 41, 41, 41, 38, 39, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939031606Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45712024-02-27T21:45:49.553Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939067372Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45722024-02-27T21:45:49.553Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939091919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45732024-02-27T21:45:49.553Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939123615Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45742024-02-27T21:45:49.553Z{"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939140373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45752024-02-27T21:45:49.553Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.93916603Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45762024-02-27T21:45:49.553Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939179969Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45772024-02-27T21:45:49.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939198417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45782024-02-27T21:45:49.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939220674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45792024-02-27T21:45:49.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939234492Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45802024-02-27T21:45:49.554Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939248471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45812024-02-27T21:45:49.554Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939273898Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45822024-02-27T21:45:49.554Z{"msg":"77df52bc-e899-4e8a-bd2d-0baf3f98e763 is now active with session: 797e0c4e-944f-45f9-8384-412f04612a48","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939292675Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45832024-02-27T21:45:49.554Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939309813Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306,"session_id":"797e0c4e-944f-45f9-8384-412f04612a48"}
45842024-02-27T21:45:49.554Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:47.939322442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306}
45852024-02-27T21:45:49.554ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45862024-02-27T21:45:49.555ZLoading write count information from file "/tmp/test_repair_verify.data"
45872024-02-27T21:45:49.555ZRun Repair workload
45882024-02-27T21:45:49.555Z01/30 Write block 39 len 16384 data: 6 6 6 5
45892024-02-27T21:45:49.555Z02/30 Read block 99 len 4096
45902024-02-27T21:45:49.555Z03/30 Flush
45912024-02-27T21:45:49.561Z04/30 Write block 178 len 28672 data: 4 2 3 4 4 4 5
45922024-02-27T21:45:49.561Z05/30 Read block 24 len 40960
45932024-02-27T21:45:49.561Z06/30 Write block 541 len 8192 data: 3 3
45942024-02-27T21:45:49.562Z07/30 Read block 575 len 20480
45952024-02-27T21:45:49.562Z08/30 Read block 278 len 8192
45962024-02-27T21:45:49.562Z09/30 Read block 334 len 8192
45972024-02-27T21:45:49.562Z10/30 Write block 194 len 40960 data: 4 3 3 3 3 3 5 5 5 5
45982024-02-27T21:45:49.562Z11/30 Read block 401 len 28672
45992024-02-27T21:45:49.562Z12/30 Read block 440 len 24576
46002024-02-27T21:45:49.563Z13/30 Write block 176 len 36864 data: 5 5 5 3 4 5 5 5 6
46012024-02-27T21:45:49.563Z14/30 Read block 422 len 12288
46022024-02-27T21:45:49.563Z15/30 Write block 385 len 36864 data: 4 4 7 6 7 6 6 5 6
46032024-02-27T21:45:49.563Z16/30 Read block 477 len 16384
46042024-02-27T21:45:49.563Z17/30 Write block 2 len 4096 data: 2
46052024-02-27T21:45:49.563Z18/30 Flush
46062024-02-27T21:45:49.564Z19/30 Write block 458 len 16384 data: 2 5 4 5
46072024-02-27T21:45:49.564Z20/30 Read block 283 len 32768
46082024-02-27T21:45:49.564Z21/30 Read block 292 len 4096
46092024-02-27T21:45:49.564Z22/30 Read block 114 len 12288
46102024-02-27T21:45:49.565Z23/30 Read block 383 len 24576
46112024-02-27T21:45:49.565Z24/30 Write block 554 len 40960 data: 6 6 5 8 7 5 5 5 5 4
46122024-02-27T21:45:49.565Z25/30 Write block 589 len 4096 data: 3
46132024-02-27T21:45:49.570Z26/30 Write block 338 len 32768 data: 5 5 4 5 6 6 6 6
46142024-02-27T21:45:49.571Z27/30 Read block 314 len 12288
46152024-02-27T21:45:49.571Z28/30 Read block 404 len 32768
46162024-02-27T21:45:49.571Z29/30 Write block 486 len 28672 data: 5 4 5 6 6 7 8
46172024-02-27T21:45:49.571Z30/30 Read block 370 len 12288
46182024-02-27T21:45:49.571Z----------------------------------------------------------------
46192024-02-27T21:45:49.571Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:27
46202024-02-27T21:45:49.571ZStates: Active Active Active
46212024-02-27T21:45:49.571ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46222024-02-27T21:45:49.571Z 4 Acked 1003 Write 7 Sent Done Done false
46232024-02-27T21:45:49.571Z 5 Acked 1004 Read 10 Sent Done Done false
46242024-02-27T21:45:49.572Z 6 Acked 1005 Write 2 Sent Done Done false
46252024-02-27T21:45:49.572Z 7 Acked 1006 Read 5 Sent Done Done false
46262024-02-27T21:45:49.572Z 8 Acked 1007 Read 2 Sent Done Done false
46272024-02-27T21:45:49.572Z 9 Acked 1008 Read 2 Sent Done Done false
46282024-02-27T21:45:49.572Z 10 Acked 1009 Write 10 Sent Done Done false
46292024-02-27T21:45:49.572Z 11 Acked 1010 Read 7 Sent Done Done false
46302024-02-27T21:45:49.572Z 12 Acked 1011 Read 6 Sent Done Done false
46312024-02-27T21:45:49.572Z 13 Acked 1012 Write 9 Sent Done Done false
46322024-02-27T21:45:49.572Z 14 Acked 1013 Read 3 Sent Done Done false
46332024-02-27T21:45:49.572Z 15 Acked 1014 Write 9 Sent Done Done false
46342024-02-27T21:45:49.573Z 16 Acked 1015 Read 4 Sent Done Done false
46352024-02-27T21:45:49.573Z 17 Acked 1016 Write 1 Sent Done Done false
46362024-02-27T21:45:49.573Z 18 Acked 1017 Flush 0 Sent Done Done false
46372024-02-27T21:45:49.573Z 19 Acked 1018 Write 4 Sent Done Done false
46382024-02-27T21:45:49.573Z 20 Acked 1019 Read 8 Sent Done Done false
46392024-02-27T21:45:49.573Z 21 Acked 1020 Read 1 Sent Done Done false
46402024-02-27T21:45:49.574Z 22 Acked 1021 Read 3 Sent Done Done false
46412024-02-27T21:45:49.574Z 23 Acked 1022 Read 6 Sent Done Done false
46422024-02-27T21:45:49.574Z 24 Acked 1023 Write 10 Sent Done Done false
46432024-02-27T21:45:49.574Z 25 Acked 1024 Write 1 Sent Done Done false
46442024-02-27T21:45:49.574Z 26 Acked 1025 Write 8 Sent Done Done false
46452024-02-27T21:45:49.574Z 27 Acked 1026 Read 3 Sent Done Done false
46462024-02-27T21:45:49.574Z 28 Acked 1027 Read 8 Sent Done Done false
46472024-02-27T21:45:49.574Z 29 Acked 1028 Write 7 Sent Done Done false
46482024-02-27T21:45:49.575Z 30 Acked 1029 Read 3 Sent Sent Done false
46492024-02-27T21:45:49.575Z STATES DS:0 DS:1 DS:2 TOTAL
46502024-02-27T21:45:49.575Z New 0 0 0 0
46512024-02-27T21:45:49.575Z Sent 27 1 0 28
46522024-02-27T21:45:49.575Z Done 0 26 27 53
46532024-02-27T21:45:49.576Z Skipped 0 0 0 0
46542024-02-27T21:45:49.576Z Error 0 0 0 0
46552024-02-27T21:45:49.576ZLast Flush: 1002 1017 1017
46562024-02-27T21:45:49.576ZDownstairs last five completed: 1002 1001 1000
46572024-02-27T21:45:49.576ZUpstairs last five completed: 30 29 28 27 26
46582024-02-27T21:45:49.576Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:47.969711492Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1306}
46592024-02-27T21:45:49.577ZWrote out file "/tmp/test_repair_verify.data"
46602024-02-27T21:45:49.577Z
46612024-02-27T21:45:49.581ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46622024-02-27T21:45:49.590ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46632024-02-27T21:45:49.590Z 0 000-019 39 42 42 68 74 74 F F F <---
46642024-02-27T21:45:49.590Z 1 020-039 42 42 42 73 73 73 F F F
46652024-02-27T21:45:49.590Z 2 040-059 42 42 42 73 73 73 F F F
46662024-02-27T21:45:49.590Z 3 060-079 37 37 37 65 65 65 F F F
46672024-02-27T21:45:49.590Z 4 080-099 41 41 41 72 72 72 F F F
46682024-02-27T21:45:49.591Z 5 100-119 41 41 41 72 72 72 F F F
46692024-02-27T21:45:49.591Z 6 120-139 41 41 41 72 72 72 F F F
46702024-02-27T21:45:49.591Z 7 140-159 41 41 41 72 72 72 F F F
46712024-02-27T21:45:49.595Z 8 160-179 38 42 42 66 74 74 F F F <---
46722024-02-27T21:45:49.595Z 9 180-199 39 42 42 68 74 74 F F F <---
46732024-02-27T21:45:49.596Z 10 200-219 38 42 42 66 74 74 F F F <---
46742024-02-27T21:45:49.596Z 11 220-239 38 38 38 66 66 66 F F F
46752024-02-27T21:45:49.596Z 12 240-259 37 37 37 65 65 65 F F F
46762024-02-27T21:45:49.596Z 13 260-279 41 41 41 72 72 72 F F F
46772024-02-27T21:45:49.596Z 14 280-299 40 40 40 69 69 69 F F F
46782024-02-27T21:45:49.596Z 15 300-319 38 38 38 66 66 66 F F F
46792024-02-27T21:45:49.596Z 16 320-339 38 38 38 66 66 66 F T T <---
46802024-02-27T21:45:49.596Z 17 340-359 39 39 39 68 68 68 F T T <---
46812024-02-27T21:45:49.597Z 18 360-379 37 37 37 65 65 65 F F F
46822024-02-27T21:45:49.597Z 19 380-399 40 42 42 70 74 74 F F F <---
46832024-02-27T21:45:49.597Z 20 400-419 40 40 40 70 70 70 F F F
46842024-02-27T21:45:49.597Z 21 420-439 41 41 41 72 72 72 F F F
46852024-02-27T21:45:49.597Z 22 440-459 35 35 35 62 62 62 F T T <---
46862024-02-27T21:45:49.597Z 23 460-479 36 36 36 63 63 63 F T T <---
46872024-02-27T21:45:49.597Z 24 480-499 40 40 40 71 71 71 F T T <---
46882024-02-27T21:45:49.597Z 25 500-519 38 38 38 67 67 67 F F F
46892024-02-27T21:45:49.597Z 26 520-539 38 38 38 66 66 66 F F F
46902024-02-27T21:45:49.597Z 27 540-559 37 42 42 65 74 74 F T T <---
46912024-02-27T21:45:49.597Z 28 560-579 41 41 41 72 72 72 F T T <---
46922024-02-27T21:45:49.597Z 29 580-599 39 39 39 68 68 68 F T T <---
46932024-02-27T21:45:49.597ZMax gen: 42, Max flush: 74
46942024-02-27T21:45:49.597ZError: Difference in extent metadata found!
46952024-02-27T21:45:49.597ZA Difference in extent metadata is expected here
46962024-02-27T21:45:49.597ZOn loop 21
46972024-02-27T21:45:49.597Z
46982024-02-27T21:45:49.606ZVerifying data now
46992024-02-27T21:45:54.301ZNo change after verify
47002024-02-27T21:45:54.301ZLoop: 21 Downstairs dump after verify (and repair):
47012024-02-27T21:45:54.318ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47022024-02-27T21:45:54.318Z 0 000-019 42 42 42 74 74 74 F F F
47032024-02-27T21:45:54.318Z 1 020-039 42 42 42 73 73 73 F F F
47042024-02-27T21:45:54.318Z 2 040-059 42 42 42 73 73 73 F F F
47052024-02-27T21:45:54.318Z 3 060-079 37 37 37 65 65 65 F F F
47062024-02-27T21:45:54.318Z 4 080-099 41 41 41 72 72 72 F F F
47072024-02-27T21:45:54.319Z 5 100-119 41 41 41 72 72 72 F F F
47082024-02-27T21:45:54.319Z 6 120-139 41 41 41 72 72 72 F F F
47092024-02-27T21:45:54.319Z 7 140-159 41 41 41 72 72 72 F F F
47102024-02-27T21:45:54.319Z 8 160-179 42 42 42 74 74 74 F F F
47112024-02-27T21:45:54.319Z 9 180-199 42 42 42 74 74 74 F F F
47122024-02-27T21:45:54.319Z 10 200-219 42 42 42 74 74 74 F F F
47132024-02-27T21:45:54.319Z 11 220-239 38 38 38 66 66 66 F F F
47142024-02-27T21:45:54.319Z 12 240-259 37 37 37 65 65 65 F F F
47152024-02-27T21:45:54.319Z 13 260-279 41 41 41 72 72 72 F F F
47162024-02-27T21:45:54.319Z 14 280-299 40 40 40 69 69 69 F F F
47172024-02-27T21:45:54.320Z 15 300-319 38 38 38 66 66 66 F F F
47182024-02-27T21:45:54.320Z 16 320-339 43 43 43 75 75 75 F F F
47192024-02-27T21:45:54.320Z 17 340-359 43 43 43 75 75 75 F F F
47202024-02-27T21:45:54.320Z 18 360-379 37 37 37 65 65 65 F F F
47212024-02-27T21:45:54.320Z 19 380-399 42 42 42 74 74 74 F F F
47222024-02-27T21:45:54.320Z 20 400-419 40 40 40 70 70 70 F F F
47232024-02-27T21:45:54.320Z 21 420-439 41 41 41 72 72 72 F F F
47242024-02-27T21:45:54.320Z 22 440-459 43 43 43 75 75 75 F F F
47252024-02-27T21:45:54.321Z 23 460-479 43 43 43 75 75 75 F F F
47262024-02-27T21:45:54.321Z 24 480-499 43 43 43 75 75 75 F F F
47272024-02-27T21:45:54.321Z 25 500-519 38 38 38 67 67 67 F F F
47282024-02-27T21:45:54.321Z 26 520-539 38 38 38 66 66 66 F F F
47292024-02-27T21:45:54.321Z 27 540-559 43 43 43 75 75 75 F F F
47302024-02-27T21:45:54.321Z 28 560-579 43 43 43 75 75 75 F F F
47312024-02-27T21:45:54.321Z 29 580-599 43 43 43 75 75 75 F F F
47322024-02-27T21:45:54.321ZMax gen: 43, Max flush: 75
47332024-02-27T21:45:54.321Z
47342024-02-27T21:45:54.321ZBegin loop 22
47352024-02-27T21:45:54.321ZDownstairs to restart: 0
47362024-02-27T21:45:54.321Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.686386406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314}
47372024-02-27T21:45:54.322Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.686672062Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314}
47382024-02-27T21:45:54.322Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.686715807Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314}
47392024-02-27T21:45:54.322Z{"msg":"Crucible 824d225b-8be1-477d-9248-fbf2da6b5ae9 has session id: 704863c8-f0d9-4d01-895f-95a4483f225e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.686743344Z","hostname":"Crucible runtime is spawned
47402024-02-27T21:45:54.322Zip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47412024-02-27T21:45:54.322Z{"msg":"Upstairs opts: Upstairs UUID: 824d225b-8be1-477d-9248-fbf2da6b5ae9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.686801117Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47422024-02-27T21:45:54.322Z{"msg":"Crucible stats registered with UUID: 824d225b-8be1-477d-9248-fbf2da6b5ae9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.686835723Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47432024-02-27T21:45:54.322Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.686868569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314}
47442024-02-27T21:45:54.323Z{"msg":"824d225b-8be1-477d-9248-fbf2da6b5ae9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.686962378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47452024-02-27T21:45:54.323Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687008432Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47462024-02-27T21:45:54.323Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687050387Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47472024-02-27T21:45:54.323Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687092013Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47482024-02-27T21:45:54.323Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687139377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47492024-02-27T21:45:54.323Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687184442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47502024-02-27T21:45:54.323Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687221957Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47512024-02-27T21:45:54.323Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687319776Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"io task","client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47522024-02-27T21:45:54.323Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687404796Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"io task","client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47532024-02-27T21:45:54.323Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687486006Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"io task","client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47542024-02-27T21:45:54.323Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687546829Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"io task","client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47552024-02-27T21:45:54.323Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.687596333Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"io task","client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47562024-02-27T21:45:54.323Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.68762653Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"io task","client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47572024-02-27T21:45:54.324Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.68804713Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47582024-02-27T21:45:54.324Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.688076417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47592024-02-27T21:45:54.324Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.688093755Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47602024-02-27T21:45:54.324Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.68813204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47612024-02-27T21:45:54.324Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.688149008Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47622024-02-27T21:45:54.324Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.688171215Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47632024-02-27T21:45:54.325Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.688597085Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47642024-02-27T21:45:54.325Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.688697424Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47652024-02-27T21:45:54.325Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.688927826Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47662024-02-27T21:45:54.325Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689259327Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47672024-02-27T21:45:54.325Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689344997Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47682024-02-27T21:45:54.325Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.68940139Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47692024-02-27T21:45:54.325Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689427227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47702024-02-27T21:45:54.325Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689451475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47712024-02-27T21:45:54.325Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689466713Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47722024-02-27T21:45:54.326Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.68948766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47732024-02-27T21:45:54.326Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689502059Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47742024-02-27T21:45:54.326Z{"msg":"[0]R flush_numbers[0..12]: [74, 73, 73, 65, 72, 72, 72, 72, 74, 74, 74, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689522716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47752024-02-27T21:45:54.326Z{"msg":"[0]R generation[0..12]: [42, 42, 42, 37, 41, 41, 41, 41, 42, 42, 42, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689537874Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47762024-02-27T21:45:54.326Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689559412Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47772024-02-27T21:45:54.326Z{"msg":"[1]R flush_numbers[0..12]: [74, 73, 73, 65, 72, 72, 72, 72, 74, 74, 74, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689595558Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47782024-02-27T21:45:54.326Z{"msg":"[1]R generation[0..12]: [42, 42, 42, 37, 41, 41, 41, 41, 42, 42, 42, 38]","v":Wait for a query_work_queue command to finish before sending IO
47792024-02-27T21:45:54.326Z0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689621105Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47802024-02-27T21:45:54.327Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.68965809Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47812024-02-27T21:45:54.327Z{"msg":"[2]R flush_numbers[0..12]: [74, 73, 73, 65, 72, 72, 72, 72, 74, 74, 74, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689691836Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47822024-02-27T21:45:54.327Z{"msg":"[2]R generation[0..12]: [42, 42, 42, 37, 41, 41, 41, 41, 42, 42, 42, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689713694Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47832024-02-27T21:45:54.327Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689738331Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47842024-02-27T21:45:54.327Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689766767Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47852024-02-27T21:45:54.327Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689784765Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47862024-02-27T21:45:54.327Z{"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689813552Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47872024-02-27T21:45:54.327Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.68982881Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47882024-02-27T21:45:54.327Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689859607Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47892024-02-27T21:45:54.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689879214Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47902024-02-27T21:45:54.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689908251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47912024-02-27T21:45:54.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689928228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47922024-02-27T21:45:54.327Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689956185Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47932024-02-27T21:45:54.327Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.689974073Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47942024-02-27T21:45:54.328Z{"msg":"824d225b-8be1-477d-9248-fbf2da6b5ae9 is now active with session: 704863c8-f0d9-4d01-895f-95a4483f225e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.690007369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47952024-02-27T21:45:54.328Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.690045065Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
47962024-02-27T21:45:54.328Zip-10-150-1-196.us-west-2.compute.internal","pid":1314,"session_id":"704863c8-f0d9-4d01-895f-95a4483f225e"}
47972024-02-27T21:45:54.328Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:52.690079481Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314}
47982024-02-27T21:45:54.328ZLoading write count information from file "/tmp/test_repair_verify.data"
47992024-02-27T21:45:54.328ZRun Repair workload
48002024-02-27T21:45:54.328Z01/30 Write block 505 len 12288 data: 4 4 4
48012024-02-27T21:45:54.342Z02/30 Flush
48022024-02-27T21:45:54.342Z03/30 Flush
48032024-02-27T21:45:54.342Z04/30 Write block 518 len 12288 data: 3 3 3
48042024-02-27T21:45:54.342Z05/30 Read block 175 len 12288
48052024-02-27T21:45:54.342Z06/30 Read block 444 len 24576
48062024-02-27T21:45:54.342Z07/30 Read block 335 len 36864
48072024-02-27T21:45:54.342Z08/30 Write block 48 len 12288 data: 2 3 4
48082024-02-27T21:45:54.342Z09/30 Flush
48092024-02-27T21:45:54.342Z10/30 Write block 338 len 28672 data: 6 6 5 6 7 7 7
48102024-02-27T21:45:54.343Z11/30 Write block 590 len 28672 data: 2 2 2 2 2 2 4
48112024-02-27T21:45:54.343Z12/30 Read block 348 len 20480
48122024-02-27T21:45:54.343Z13/30 Write block 302 len 4096 data: 4
48132024-02-27T21:45:54.343Z14/30 Flush
48142024-02-27T21:45:54.343Z15/30 Write block 232 len 4096 data: 10
48152024-02-27T21:45:54.343Z16/30 Read block 255 len 24576
48162024-02-27T21:45:54.343Z17/30 Write block 272 len 4096 data: 4
48172024-02-27T21:45:54.343Z18/30 Read block 206 len 20480
48182024-02-27T21:45:54.343Z19/30 Write block 56 len 16384 data: 6 6 6 5
48192024-02-27T21:45:54.343Z20/30 Write block 498 len 16384 data: 6 6 5 3
48202024-02-27T21:45:54.343Z21/30 Write block 206 len 8192 data: 3 4
48212024-02-27T21:45:54.343Z22/30 Write block 167 len 32768 data: 2 3 4 4 3 5 5 6
48222024-02-27T21:45:54.343Z23/30 Flush
48232024-02-27T21:45:54.343Z24/30 Write block 308 len 16384 data: 9 10 10 10
48242024-02-27T21:45:54.343Z25/30 Write block 286 len 28672 data: 4 4 4 7 6 5 5
48252024-02-27T21:45:54.343Z26/30 Read block 518 len 28672
48262024-02-27T21:45:54.343Z27/30 Write block 486 len 16384 data: 6 5 6 7
48272024-02-27T21:45:54.343Z28/30 Write block 489 len 8192 data: 8 7
48282024-02-27T21:45:54.343Z29/30 Read block 326 len 16384
48292024-02-27T21:45:54.344Z30/30 Read block 167 len 24576
48302024-02-27T21:45:54.344Z----------------------------------------------------------------
48312024-02-27T21:45:54.344Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48322024-02-27T21:45:54.344ZStates: Active Active Active
48332024-02-27T21:45:54.344ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48342024-02-27T21:45:54.344Z 1 Acked 1000 Write 3 Sent Done Done false
48352024-02-27T21:45:54.344Z 2 Acked 1001 Flush 0 Sent Done Done false
48362024-02-27T21:45:54.344Z 3 Acked 1002 Flush 0 Sent Done Done false
48372024-02-27T21:45:54.344Z 4 Acked 1003 Write 3 Sent Done Done false
48382024-02-27T21:45:54.344Z 5 Acked 1004 Read 3 Sent Done Done false
48392024-02-27T21:45:54.344Z 6 Acked 1005 Read 6 Sent Done Done false
48402024-02-27T21:45:54.344Z 7 Acked 1006 Read 9 Sent Done Done false
48412024-02-27T21:45:54.344Z 8 Acked 1007 Write 3 Sent Done Done false
48422024-02-27T21:45:54.344Z 9 Acked 1008 Flush 0 Sent Done Done false
48432024-02-27T21:45:54.344Z 10 Acked 1009 Write 7 Sent Done Done false
48442024-02-27T21:45:54.344Z 11 Acked 1010 Write 7 Sent Done Done false
48452024-02-27T21:45:54.344Z 12 Acked 1011 Read 5 Sent Done Done false
48462024-02-27T21:45:54.344Z 13 Acked 1012 Write 1 Sent Done Done false
48472024-02-27T21:45:54.345Z 14 Acked 1013 Flush 0 Sent Done Done false
48482024-02-27T21:45:54.345Z 15 Acked 1014 Write 1 Sent Done Done false
48492024-02-27T21:45:54.345Z 16 Acked 1015 Read 6 Sent Done Done false
48502024-02-27T21:45:54.345Z 17 Acked 1016 Write 1 Sent Done Done false
48512024-02-27T21:45:54.345Z 18 Acked 1017 Read 5 Sent Done Done false
48522024-02-27T21:45:54.345Z 19 Acked 1018 Write 4 Sent Done Done false
48532024-02-27T21:45:54.345Z 20 Acked 1019 Write 4 Sent Done Done false
48542024-02-27T21:45:54.345Z 21 Acked 1020 Write 2 Sent Done Done false
48552024-02-27T21:45:54.345Z 22 Acked 1021 Write 8 Sent Done Done false
48562024-02-27T21:45:54.345Z 23 Acked 1022 Flush 0 Sent Done Done false
48572024-02-27T21:45:54.345Z 24 Acked 1023 Write 4 Sent Done Done false
48582024-02-27T21:45:54.345Z 25 Acked 1024 Write 7 Sent Done Done false
48592024-02-27T21:45:54.345Z 26 Acked 1025 Read 7 Sent Done Done false
48602024-02-27T21:45:54.345Z 27 Acked 1026 Write 4 Sent Done Done false
48612024-02-27T21:45:54.345Z 28 Acked 1027 Write 2 Sent Done Done false
48622024-02-27T21:45:54.345Z 29 Acked 1028 Read 4 Sent Done Done false
48632024-02-27T21:45:54.345Z 30 Acked 1029 Read 6 Sent Sent Done false
48642024-02-27T21:45:54.345Z STATES DS:0 DS:1 DS:2 TOTAL
48652024-02-27T21:45:54.345Z New 0 0 0 0
48662024-02-27T21:45:54.345Z Sent 30 1 0 31
48672024-02-27T21:45:54.345Z Done 0 29 30 59
48682024-02-27T21:45:54.345Z Skipped 0 0 0 0
48692024-02-27T21:45:54.346Z Error 0 0 0 0
48702024-02-27T21:45:54.346ZLast Flush: 0 1022 1022
48712024-02-27T21:45:54.346ZDownstairs last five completed:
48722024-02-27T21:45:54.346ZUpstairs last five completed: 30 29 28 27 26
48732024-02-27T21:45:54.346Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:52.71988657Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1314}
48742024-02-27T21:45:54.346ZWrote out file "/tmp/test_repair_verify.data"
48752024-02-27T21:45:54.346Z
48762024-02-27T21:45:54.346ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48772024-02-27T21:45:54.357ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48782024-02-27T21:45:54.357Z 0 000-019 42 42 42 74 74 74 F F F
48792024-02-27T21:45:54.358Z 1 020-039 42 42 42 73 73 73 F F F
48802024-02-27T21:45:54.358Z 2 040-059 42 44 44 73 80 80 F F F <---
48812024-02-27T21:45:54.358Z 3 060-079 37 37 37 65 65 65 F F F
48822024-02-27T21:45:54.359Z 4 080-099 41 41 41 72 72 72 F F F
48832024-02-27T21:45:54.359Z 5 100-119 41 41 41 72 72 72 F F F
48842024-02-27T21:45:54.359Z 6 120-139 41 41 41 72 72 72 F F F
48852024-02-27T21:45:54.359Z 7 140-159 41 41 41 72 72 72 F F F
48862024-02-27T21:45:54.360Z 8 160-179 42 44 44 74 80 80 F F F <---
48872024-02-27T21:45:54.360Z 9 180-199 42 42 42 74 74 74 F F F
48882024-02-27T21:45:54.360Z 10 200-219 42 44 44 74 80 80 F F F <---
48892024-02-27T21:45:54.360Z 11 220-239 38 44 44 66 80 80 F F F <---
48902024-02-27T21:45:54.360Z 12 240-259 37 37 37 65 65 65 F F F
48912024-02-27T21:45:54.360Z 13 260-279 41 44 44 72 80 80 F F F <---
48922024-02-27T21:45:54.361Z 14 280-299 40 40 40 69 69 69 F T T <---
48932024-02-27T21:45:54.361Z 15 300-319 38 44 44 66 79 79 F T T <---
48942024-02-27T21:45:54.361Z 16 320-339 43 44 44 75 79 79 F F F <---
48952024-02-27T21:45:54.361Z 17 340-359 43 44 44 75 79 79 F F F <---
48962024-02-27T21:45:54.361Z 18 360-379 37 37 37 65 65 65 F F F
48972024-02-27T21:45:54.361Z 19 380-399 42 42 42 74 74 74 F F F
48982024-02-27T21:45:54.361Z 20 400-419 40 40 40 70 70 70 F F F
48992024-02-27T21:45:54.361Z 21 420-439 41 41 41 72 72 72 F F F
49002024-02-27T21:45:54.362Z 22 440-459 43 43 43 75 75 75 F F F
49012024-02-27T21:45:54.362Z 23 460-479 43 43 43 75 75 75 F F F
49022024-02-27T21:45:54.362Z 24 480-499 43 44 44 75 80 80 F T T <---
49032024-02-27T21:45:54.362Z 25 500-519 38 44 44 67 80 80 F F F <---
49042024-02-27T21:45:54.362Z 26 520-539 38 44 44 66 78 78 F F F <---
49052024-02-27T21:45:54.363Z 27 540-559 43 43 43 75 75 75 F F F
49062024-02-27T21:45:54.363Z 28 560-579 43 43 43 75 75 75 F F F
49072024-02-27T21:45:54.363Z 29 580-599 43 44 44 75 79 79 F F F <---
49082024-02-27T21:45:54.363ZMax gen: 44, Max flush: 80
49092024-02-27T21:45:54.363ZError: Difference in extent metadata found!
49102024-02-27T21:45:54.363ZA Difference in extent metadata is expected here
49112024-02-27T21:45:54.363ZOn loop 22
49122024-02-27T21:45:54.363Z
49132024-02-27T21:45:54.363ZVerifying data now
49142024-02-27T21:45:59.808ZNo change after verify
49152024-02-27T21:45:59.808ZLoop: 22 Downstairs dump after verify (and repair):
49162024-02-27T21:45:59.945ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49172024-02-27T21:45:59.945Z 0 000-019 42 42 42 74 74 74 F F F
49182024-02-27T21:45:59.945Z 1 020-039 42 42 42 73 73 73 F F F
49192024-02-27T21:45:59.945Z 2 040-059 44 44 44 80 80 80 F F F
49202024-02-27T21:45:59.946Z 3 060-079 37 37 37 65 65 65 F F F
49212024-02-27T21:45:59.946Z 4 080-099 41 41 41 72 72 72 F F F
49222024-02-27T21:45:59.946Z 5 100-119 41 41 41 72 72 72 F F F
49232024-02-27T21:45:59.946Z 6 120-139 41 41 41 72 72 72 F F F
49242024-02-27T21:45:59.946Z 7 140-159 41 41 41 72 72 72 F F F
49252024-02-27T21:45:59.946Z 8 160-179 44 44 44 80 80 80 F F F
49262024-02-27T21:45:59.946Z 9 180-199 42 42 42 74 74 74 F F F
49272024-02-27T21:45:59.946Z 10 200-219 44 44 44 80 80 80 F F F
49282024-02-27T21:45:59.946Z 11 220-239 44 44 44 80 80 80 F F F
49292024-02-27T21:45:59.946Z 12 240-259 37 37 37 65 65 65 F F F
49302024-02-27T21:45:59.946Z 13 260-279 44 44 44 80 80 80 F F F
49312024-02-27T21:45:59.947Z 14 280-299 45 45 45 81 81 81 F F F
49322024-02-27T21:45:59.947Z 15 300-319 45 45 45 81 81 81 F F F
49332024-02-27T21:45:59.947Z 16 320-339 44 44 44 79 79 79 F F F
49342024-02-27T21:45:59.947Z 17 340-359 44 44 44 79 79 79 F F F
49352024-02-27T21:45:59.947Z 18 360-379 37 37 37 65 65 65 F F F
49362024-02-27T21:45:59.947Z 19 380-399 42 42 42 74 74 74 F F F
49372024-02-27T21:45:59.947Z 20 400-419 40 40 40 70 70 70 F F F
49382024-02-27T21:45:59.947Z 21 420-439 41 41 41 72 72 72 F F F
49392024-02-27T21:45:59.947Z 22 440-459 43 43 43 75 75 75 F F F
49402024-02-27T21:45:59.947Z 23 460-479 43 43 43 75 75 75 F F F
49412024-02-27T21:45:59.947Z 24 480-499 45 45 45 81 81 81 F F F
49422024-02-27T21:45:59.947Z 25 500-519 44 44 44 80 80 80 F F F
49432024-02-27T21:45:59.947Z 26 520-539 44 44 44 78 78 78 F F F
49442024-02-27T21:45:59.947Z 27 540-559 43 43 43 75 75 75 F F F
49452024-02-27T21:45:59.947Z 28 560-579 43 43 43 75 75 75 F F F
49462024-02-27T21:45:59.947Z 29 580-599 44 44 44 79 79 79 F F F
49472024-02-27T21:45:59.947ZMax gen: 45, Max flush: 81
49482024-02-27T21:45:59.947Z
49492024-02-27T21:45:59.947ZBegin loop 23
49502024-02-27T21:45:59.947ZDownstairs to restart: 0
49512024-02-27T21:45:59.948Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408194344Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322}
49522024-02-27T21:45:59.948Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"Crucible runtime is spawned
49532024-02-27T21:45:59.948Zx86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408522415Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322}
49542024-02-27T21:45:59.948Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408555741Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322}
49552024-02-27T21:45:59.948Z{"msg":"Crucible 9c9acd82-8f64-4a61-b26f-877a764519c8 has session id: 294b4af1-1d3e-4164-8d0d-4329e8e4f4e9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408580108Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49562024-02-27T21:45:59.948Z{"msg":"Upstairs opts: Upstairs UUID: 9c9acd82-8f64-4a61-b26f-877a764519c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408633582Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49572024-02-27T21:45:59.948Z{"msg":"Crucible stats registered with UUID: 9c9acd82-8f64-4a61-b26f-877a764519c8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408674477Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49582024-02-27T21:45:59.948Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408711543Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322}
49592024-02-27T21:45:59.948Z{"msg":"9c9acd82-8f64-4a61-b26f-877a764519c8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408759847Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49602024-02-27T21:45:59.948Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408801722Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49612024-02-27T21:45:59.948Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408866084Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49622024-02-27T21:45:59.948Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.40890384Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49632024-02-27T21:45:59.948Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.408935606Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49642024-02-27T21:45:59.948Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.409062651Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49652024-02-27T21:45:59.948Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.409096177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49662024-02-27T21:45:59.948Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.409120794Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"io task","client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49672024-02-27T21:45:59.948Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.409177258Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"io task","client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49682024-02-27T21:45:59.948Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.409206404Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"io task","client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49692024-02-27T21:45:59.948Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.40924636Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"io task","client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49702024-02-27T21:45:59.948Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.409271577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"io task","client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49712024-02-27T21:45:59.948Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.409312682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"io task","client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49722024-02-27T21:45:59.948Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.409879645Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49732024-02-27T21:45:59.948Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.409915421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49742024-02-27T21:45:59.949Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.410037706Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49752024-02-27T21:45:59.949Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.410087281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49762024-02-27T21:45:59.949Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.410114577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49772024-02-27T21:45:59.949Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.410164441Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49782024-02-27T21:45:59.949Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.410444088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49792024-02-27T21:45:59.949Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.410582432Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49802024-02-27T21:45:59.949Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.41085242Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49812024-02-27T21:45:59.949Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411019261Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49822024-02-27T21:45:59.949Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411067155Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49832024-02-27T21:45:59.949Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411102901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49842024-02-27T21:45:59.949Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411134777Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49852024-02-27T21:45:59.949Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411168633Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49862024-02-27T21:45:59.949Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411202319Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49872024-02-27T21:45:59.949Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411402156Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49882024-02-27T21:45:59.949Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.41144613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49892024-02-27T21:45:59.949Z{"msg":"[0]R flush_numbers[0..12]: [74, 73, 80, 65, 72, 72, 72, 72, 80, 74, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411475217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49902024-02-27T21:45:59.949Z{"msg":"[0]R generation[0..12]: [42, 42, 44, 37, 41, 41, 41, 41, 44, 42, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411506743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49912024-02-27T21:45:59.949Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.41153405Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49922024-02-27T21:45:59.949ZWait for a query_work_queue command to finish before sending IO
49932024-02-27T21:45:59.949Z{"msg":"[1]R flush_numbers[0..12]: [74, 73, 80, 65, 72, 72, 72, 72, 80, 74, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411566536Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49942024-02-27T21:45:59.949Z{"msg":"[1]R generation[0..12]: [42, 42, 44, 37, 41, 41, 41, 41, 44, 42, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411593263Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49952024-02-27T21:45:59.949Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.41161894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49962024-02-27T21:45:59.949Z{"msg":"[2]R flush_numbers[0..12]: [74, 73, 80, 65, 72, 72, 72, 72, 80, 74, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411651696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49972024-02-27T21:45:59.950Z{"msg":"[2]R generation[0..12]: [42, 42, 44, 37, 41, 41, 41, 41, 44, 42, 44, 44]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411678953Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49982024-02-27T21:45:59.950Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411709339Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
49992024-02-27T21:45:59.950Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411734176Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50002024-02-27T21:45:59.950Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411763653Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50012024-02-27T21:45:59.950Z{"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.41178697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50022024-02-27T21:45:59.950Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411815207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50032024-02-27T21:45:59.950Z{"msg":"No downstairs reconciliation required","v":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50042024-02-27T21:45:59.950Z0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411855562Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50052024-02-27T21:45:59.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411879079Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50062024-02-27T21:45:59.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411907216Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50072024-02-27T21:45:59.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411939772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50082024-02-27T21:45:59.950Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.4119632Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50092024-02-27T21:45:59.950Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.411985097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50102024-02-27T21:45:59.950Z{"msg":"9c9acd82-8f64-4a61-b26f-877a764519c8 is now active with session: 294b4af1-1d3e-4164-8d0d-4329e8e4f4e9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.412016343Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50112024-02-27T21:45:59.950Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.412038951Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322,"session_id":"294b4af1-1d3e-4164-8d0d-4329e8e4f4e9"}
50122024-02-27T21:45:59.950Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:45:57.412065628Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322}
50132024-02-27T21:45:59.950ZLoading write count information from file "/tmp/test_repair_verify.data"
50142024-02-27T21:45:59.950ZRun Repair workload
50152024-02-27T21:45:59.950Z01/30 Flush
50162024-02-27T21:46:00.025Z02/30 Write block 255 len 36864 data: 4 4 2 4 4 5 5 4 3
50172024-02-27T21:46:00.025Z03/30 Flush
50182024-02-27T21:46:00.025Z04/30 Write block 56 len 8192 data: 7 7
50192024-02-27T21:46:00.026Z05/30 Read block 187 len 12288
50202024-02-27T21:46:00.026Z06/30 Read block 200 len 40960
50212024-02-27T21:46:00.026Z07/30 Read block 543 len 24576
50222024-02-27T21:46:00.026Z08/30 Read block 251 len 12288
50232024-02-27T21:46:00.026Z09/30 Flush
50242024-02-27T21:46:00.026Z10/30 Flush
50252024-02-27T21:46:00.026Z11/30 Write block 557 len 4096 data: 9
50262024-02-27T21:46:00.026Z12/30 Read block 513 len 12288
50272024-02-27T21:46:00.026Z13/30 Flush
50282024-02-27T21:46:00.026Z14/30 Write block 330 len 24576 data: 5 5 6 6 6 7
50292024-02-27T21:46:00.027Z15/30 Write block 217 len 32768 data: 5 6 5 5 6 8 7 8
50302024-02-27T21:46:00.027Z16/30 Flush
50312024-02-27T21:46:00.027Z17/30 Write block 215 len 28672 data: 2 3 6 7 6 6 7
50322024-02-27T21:46:00.027Z18/30 Write block 9 len 8192 data: 5 5
50332024-02-27T21:46:00.027Z19/30 Write block 39 len 8192 data: 7 7
50342024-02-27T21:46:00.027Z20/30 Read block 262 len 12288
50352024-02-27T21:46:00.027Z21/30 Read block 460 len 20480
50362024-02-27T21:46:00.028Z22/30 Write block 482 len 28672 data: 7 7 7 7 7 6 7
50372024-02-27T21:46:00.028Z23/30 Flush
50382024-02-27T21:46:00.028Z24/30 Write block 533 len 12288 data: 2 2 2
50392024-02-27T21:46:00.028Z25/30 Write block 464 len 24576 data: 5 6 6 5 3 2
50402024-02-27T21:46:00.028Z26/30 Flush
50412024-02-27T21:46:00.028Z27/30 Write block 283 len 4096 data: 4
50422024-02-27T21:46:00.028Z28/30 Write block 300 len 36864 data: 3 3 5 4 7 6 7 9 10
50432024-02-27T21:46:00.028Z29/30 Write block 383 len 28672 data: 2 3 5 5 8 7 8
50442024-02-27T21:46:00.029Z30/30 Read block 291 len 32768
50452024-02-27T21:46:00.029Z----------------------------------------------------------------
50462024-02-27T21:46:00.029Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:27
50472024-02-27T21:46:00.029ZStates: Active Active Active
50482024-02-27T21:46:00.029ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50492024-02-27T21:46:00.030Z 4 Acked 1003 Write 2 Done Done Done false
50502024-02-27T21:46:00.030Z 5 Acked 1004 Read 3 Sent Done Done false
50512024-02-27T21:46:00.030Z 6 Acked 1005 Read 10 Sent Done Done false
50522024-02-27T21:46:00.030Z 7 Acked 1006 Read 6 Sent Done Done false
50532024-02-27T21:46:00.030Z 8 Acked 1007 Read 3 Sent Done Done false
50542024-02-27T21:46:00.030Z 9 Acked 1008 Flush 0 Sent Done Done false
50552024-02-27T21:46:00.030Z 10 Acked 1009 Flush 0 Sent Done Done false
50562024-02-27T21:46:00.031Z 11 Acked 1010 Write 1 Sent Done Done false
50572024-02-27T21:46:00.031Z 12 Acked 1011 Read 3 Sent Done Done false
50582024-02-27T21:46:00.031Z 13 Acked 1012 Flush 0 Sent Done Done false
50592024-02-27T21:46:00.031Z 14 Acked 1013 Write 6 Sent Done Done false
50602024-02-27T21:46:00.031Z 15 Acked 1014 Write 8 Sent Done Done false
50612024-02-27T21:46:00.031Z 16 Acked 1015 Flush 0 Sent Done Done false
50622024-02-27T21:46:00.031Z 17 Acked 1016 Write 7 Sent Done Done false
50632024-02-27T21:46:00.031Z 18 Acked 1017 Write 2 Sent Done Done false
50642024-02-27T21:46:00.031Z 19 Acked 1018 Write 2 Sent Done Done false
50652024-02-27T21:46:00.031Z 20 Acked 1019 Read 3 Sent Done Done false
50662024-02-27T21:46:00.031Z 21 Acked 1020 Read 5 Sent Done Done false
50672024-02-27T21:46:00.031Z 22 Acked 1021 Write 7 Sent Done Done false
50682024-02-27T21:46:00.031Z 23 Acked 1022 Flush 0 Sent Done Done false
50692024-02-27T21:46:00.032Z 24 Acked 1023 Write 3 Sent Done Done false
50702024-02-27T21:46:00.032Z 25 Acked 1024 Write 6 Sent Done Done false
50712024-02-27T21:46:00.033Z 26 Acked 1025 Flush 0 Sent Done Done false
50722024-02-27T21:46:00.033Z 27 Acked 1026 Write 1 Sent Done Done false
50732024-02-27T21:46:00.033Z 28 Acked 1027 Write 9 Sent Done Done false
50742024-02-27T21:46:00.033Z 29 Acked 1028 Write 7 Sent Done Done false
50752024-02-27T21:46:00.033Z 30 Acked 1029 Read 8 Sent Done Done false
50762024-02-27T21:46:00.033Z STATES DS:0 DS:1 DS:2 TOTAL
50772024-02-27T21:46:00.033Z New 0 0 0 0
50782024-02-27T21:46:00.033Z Sent 26 0 0 26
50792024-02-27T21:46:00.033Z Done 1 27 27 55
50802024-02-27T21:46:00.033Z Skipped 0 0 0 0
50812024-02-27T21:46:00.034Z Error 0 0 0 0
50822024-02-27T21:46:00.034ZLast Flush: 1002 1025 1025
50832024-02-27T21:46:00.034ZDownstairs last five completed: 1002 1001 1000
50842024-02-27T21:46:00.034ZUpstairs last five completed: 30 29 28 27 26
50852024-02-27T21:46:00.034Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:45:57.445439116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1322}
50862024-02-27T21:46:00.034ZWrote out file "/tmp/test_repair_verify.data"
50872024-02-27T21:46:00.034Z
50882024-02-27T21:46:00.034ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
50892024-02-27T21:46:00.034ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
50902024-02-27T21:46:00.034Z 0 000-019 42 46 46 74 88 88 F F F <---
50912024-02-27T21:46:00.034Z 1 020-039 42 46 46 73 88 88 F F F <---
50922024-02-27T21:46:00.034Z 2 040-059 44 46 46 80 88 88 T F F <---
50932024-02-27T21:46:00.034Z 3 060-079 37 37 37 65 65 65 F F F
50942024-02-27T21:46:00.034Z 4 080-099 41 41 41 72 72 72 F F F
50952024-02-27T21:46:00.034Z 5 100-119 41 41 41 72 72 72 F F F
50962024-02-27T21:46:00.034Z 6 120-139 41 41 41 72 72 72 F F F
50972024-02-27T21:46:00.034Z 7 140-159 41 41 41 72 72 72 F F F
50982024-02-27T21:46:00.034Z 8 160-179 44 44 44 80 80 80 F F F
50992024-02-27T21:46:00.034Z 9 180-199 42 42 42 74 74 74 F F F
51002024-02-27T21:46:00.034Z 10 200-219 44 46 46 80 88 88 F F F <---
51012024-02-27T21:46:00.034Z 11 220-239 44 46 46 80 88 88 F F F <---
51022024-02-27T21:46:00.034Z 12 240-259 46 46 46 83 83 83 F F F
51032024-02-27T21:46:00.034Z 13 260-279 46 46 46 83 83 83 F F F
51042024-02-27T21:46:00.034Z 14 280-299 45 45 45 81 81 81 F T T <---
51052024-02-27T21:46:00.034Z 15 300-319 45 45 45 81 81 81 F T T <---
51062024-02-27T21:46:00.035Z 16 320-339 44 46 46 79 87 87 F F F <---
51072024-02-27T21:46:00.035Z 17 340-359 44 44 44 79 79 79 F F F
51082024-02-27T21:46:00.035Z 18 360-379 37 37 37 65 65 65 F F F
51092024-02-27T21:46:00.035Z 19 380-399 42 42 42 74 74 74 F T T <---
51102024-02-27T21:46:00.035Z 20 400-419 40 40 40 70 70 70 F F F
51112024-02-27T21:46:00.035Z 21 420-439 41 41 41 72 72 72 F F F
51122024-02-27T21:46:00.035Z 22 440-459 43 43 43 75 75 75 F F F
51132024-02-27T21:46:00.035Z 23 460-479 43 46 46 75 89 89 F F F <---
51142024-02-27T21:46:00.035Z 24 480-499 45 46 46 81 88 88 F F F <---
51152024-02-27T21:46:00.035Z 25 500-519 44 44 44 80 80 80 F F F
51162024-02-27T21:46:00.132Z 26 520-539 44 46 46 78 89 89 F F F <---
51172024-02-27T21:46:00.133Z 27 540-559 43 46 46 75 86 86 F F F <---
51182024-02-27T21:46:00.133Z 28 560-579 43 43 43 75 75 75 F F F
51192024-02-27T21:46:00.133Z 29 580-599 44 44 44 79 79 79 F F F
51202024-02-27T21:46:00.133ZMax gen: 46, Max flush: 89
51212024-02-27T21:46:00.133ZError: Difference in extent metadata found!
51222024-02-27T21:46:00.133ZA Difference in extent metadata is expected here
51232024-02-27T21:46:00.134ZOn loop 23
51242024-02-27T21:46:00.134Z
51252024-02-27T21:46:00.134ZVerifying data now
51262024-02-27T21:46:03.710ZNo change after verify
51272024-02-27T21:46:03.710ZLoop: 23 Downstairs dump after verify (and repair):
51282024-02-27T21:46:03.730ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51292024-02-27T21:46:03.730Z 0 000-019 46 46 46 88 88 88 F F F
51302024-02-27T21:46:03.730Z 1 020-039 46 46 46 88 88 88 F F F
51312024-02-27T21:46:03.730Z 2 040-059 46 46 46 88 88 88 F F F
51322024-02-27T21:46:03.730Z 3 060-079 37 37 37 65 65 65 F F F
51332024-02-27T21:46:03.730Z 4 080-099 41 41 41 72 72 72 F F F
51342024-02-27T21:46:03.730Z 5 100-119 41 41 41 72 72 72 F F F
51352024-02-27T21:46:03.730Z 6 120-139 41 41 41 72 72 72 F F F
51362024-02-27T21:46:03.730Z 7 140-159 41 41 41 72 72 72 F F F
51372024-02-27T21:46:03.739Z 8 160-179 44 44 44 80 80 80 F F F
51382024-02-27T21:46:03.739Z 9 180-199 42 42 42 74 74 74 F F F
51392024-02-27T21:46:03.739Z 10 200-219 46 46 46 88 88 88 F F F
51402024-02-27T21:46:03.739Z 11 220-239 46 46 46 88 88 88 F F F
51412024-02-27T21:46:03.739Z 12 240-259 46 46 46 83 83 83 F F F
51422024-02-27T21:46:03.739Z 13 260-279 46 46 46 83 83 83 F F F
51432024-02-27T21:46:03.739Z 14 280-299 47 47 47 90 90 90 F F F
51442024-02-27T21:46:03.739Z 15 300-319 47 47 47 90 90 90 F F F
51452024-02-27T21:46:03.739Z 16 320-339 46 46 46 87 87 87 F F F
51462024-02-27T21:46:03.739Z 17 340-359 44 44 44 79 79 79 F F F
51472024-02-27T21:46:03.739Z 18 360-379 37 37 37 65 65 65 F F F
51482024-02-27T21:46:03.739Z 19 380-399 47 47 47 90 90 90 F F F
51492024-02-27T21:46:03.739Z 20 400-419 40 40 40 70 70 70 F F F
51502024-02-27T21:46:03.739Z 21 420-439 41 41 41 72 72 72 F F F
51512024-02-27T21:46:03.739Z 22 440-459 43 43 43 75 75 75 F F F
51522024-02-27T21:46:03.739Z 23 460-479 46 46 46 89 89 89 F F F
51532024-02-27T21:46:03.739Z 24 480-499 46 46 46 88 88 88 F F F
51542024-02-27T21:46:03.739Z 25 500-519 44 44 44 80 80 80 F F F
51552024-02-27T21:46:03.739Z 26 520-539 46 46 46 89 89 89 F F F
51562024-02-27T21:46:03.739Z 27 540-559 46 46 46 86 86 86 F F F
51572024-02-27T21:46:03.740Z 28 560-579 43 43 43 75 75 75 F F F
51582024-02-27T21:46:03.740Z 29 580-599 44 44 44 79 79 79 F F F
51592024-02-27T21:46:03.740ZMax gen: 47, Max flush: 90
51602024-02-27T21:46:03.740Z
51612024-02-27T21:46:03.740ZBegin loop 24
51622024-02-27T21:46:03.740ZDownstairs to restart: 2
51632024-02-27T21:46:03.746Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.14775207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330}
51642024-02-27T21:46:03.746Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
51652024-02-27T21:46:04.617Z,"v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148088281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330}
51662024-02-27T21:46:04.617Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148146004Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330}
51672024-02-27T21:46:04.617Z{"msg":"Crucible 5132b473-2432-41f2-8cbe-35b96462c5ca has session id: b8e26134-c426-4401-95ad-0b29eb749d9d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.14817724Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51682024-02-27T21:46:04.617Z{"msg":"Upstairs opts: Upstairs UUID: 5132b473-2432-41f2-8cbe-35b96462c5ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148229054Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51692024-02-27T21:46:04.617Z{"msg":"Crucible stats registered with UUID: 5132b473-2432-41f2-8cbe-35b96462c5ca","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148281168Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51702024-02-27T21:46:04.618Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148309665Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330}
51712024-02-27T21:46:04.618Z{"msg":"5132b473-2432-41f2-8cbe-35b96462c5ca active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148359969Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51722024-02-27T21:46:04.618Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148393795Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51732024-02-27T21:46:04.618Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148441669Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51742024-02-27T21:46:04.618Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148482064Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51752024-02-27T21:46:04.618Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.14851725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51762024-02-27T21:46:04.618Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148533368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51772024-02-27T21:46:04.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148557026Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51782024-02-27T21:46:04.619Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148581763Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"io task","client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51792024-02-27T21:46:04.619Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148616948Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"io task","client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51802024-02-27T21:46:04.619Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148642036Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"io task","client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51812024-02-27T21:46:04.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148834723Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"io task","client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51822024-02-27T21:46:04.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148877988Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"io task","client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51832024-02-27T21:46:04.619Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.148908464Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"io task","client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51842024-02-27T21:46:04.619Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.149497745Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51852024-02-27T21:46:04.620Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.149555688Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51862024-02-27T21:46:04.620Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.149604232Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51872024-02-27T21:46:04.620Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.149651917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51882024-02-27T21:46:04.620Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.14970542Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51892024-02-27T21:46:04.620Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.149753885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51902024-02-27T21:46:04.620Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150105713Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51912024-02-27T21:46:04.620Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150144519Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51922024-02-27T21:46:04.620Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150176885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51932024-02-27T21:46:04.620Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150663987Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51942024-02-27T21:46:04.620Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150720231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51952024-02-27T21:46:04.620Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150756566Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51962024-02-27T21:46:04.620Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150780753Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51972024-02-27T21:46:04.621Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.1508089Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51982024-02-27T21:46:04.621Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150833797Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
51992024-02-27T21:46:04.621Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150857324Z","hostname":"Wait for a query_work_queue command to finish before sending IO
52002024-02-27T21:46:04.621Zip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52012024-02-27T21:46:04.621Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150883041Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52022024-02-27T21:46:04.621Z{"msg":"[0]R flush_numbers[0..12]: [88, 88, 88, 65, 72, 72, 72, 72, 80, 74, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150916797Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52032024-02-27T21:46:04.621Z{"msg":"[0]R generation[0..12]: [46, 46, 46, 37, 41, 41, 41, 41, 44, 42, 46, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150942834Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52042024-02-27T21:46:04.621Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150968921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52052024-02-27T21:46:04.621Z{"msg":"[1]R flush_numbers[0..12]: [88, 88, 88, 65, 72, 72, 72, 72, 80, 74, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.150998038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52062024-02-27T21:46:04.621Z{"msg":"[1]R generation[0..12]: [46, 46, 46, 37, 41, 41, 41, 41, 44, 42, 46, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151023475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52072024-02-27T21:46:04.621Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151052421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52082024-02-27T21:46:04.622Z{"msg":"[2]R flush_numbers[0..12]: [88, 88, 88, 65, 72, 72, 72, 72, 80, 74, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151078529Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52092024-02-27T21:46:04.622Z{"msg":"[2]R generation[0..12]: [46, 46, 46, 37, 41, 41, 41, 41, 44, 42, 46, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151107645Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52102024-02-27T21:46:04.622Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151133812Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52112024-02-27T21:46:04.622Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151161169Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52122024-02-27T21:46:04.622Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151185426Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52132024-02-27T21:46:04.622Z{"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151211073Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52142024-02-27T21:46:04.622Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.1512349Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52152024-02-27T21:46:04.622Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52162024-02-27T21:46:04.622Z,"time":"2024-02-27T21:46:02.151261127Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52172024-02-27T21:46:04.622Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151285744Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52182024-02-27T21:46:04.622Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151312281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52192024-02-27T21:46:04.622Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151336258Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52202024-02-27T21:46:04.623Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151362945Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52212024-02-27T21:46:04.623Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151385152Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52222024-02-27T21:46:04.623Z{"msg":"5132b473-2432-41f2-8cbe-35b96462c5ca is now active with session: b8e26134-c426-4401-95ad-0b29eb749d9d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151412449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52232024-02-27T21:46:04.623Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151434287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330,"session_id":"b8e26134-c426-4401-95ad-0b29eb749d9d"}
52242024-02-27T21:46:04.623Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:02.151460013Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330}
52252024-02-27T21:46:04.623ZLoading write count information from file "/tmp/test_repair_verify.data"
52262024-02-27T21:46:04.623ZRun Repair workload
52272024-02-27T21:46:04.623Z01/30 Flush
52282024-02-27T21:46:04.623Z02/30 Write block 359 len 12288 data: 6 6 3
52292024-02-27T21:46:04.623Z03/30 Write block 408 len 12288 data: 3 4 3
52302024-02-27T21:46:04.623Z04/30 Write block 59 len 4096 data: 6
52312024-02-27T21:46:04.623Z05/30 Read block 596 len 8192
52322024-02-27T21:46:04.624Z06/30 Read block 387 len 28672
52332024-02-27T21:46:04.624Z07/30 Read block 347 len 16384
52342024-02-27T21:46:04.624Z08/30 Flush
52352024-02-27T21:46:04.624Z09/30 Flush
52362024-02-27T21:46:04.625Z10/30 Write block 174 len 40960 data: 7 6 6 6 6 4 5 6 6 6
52372024-02-27T21:46:04.625Z11/30 Write block 29 len 32768 data: 6 7 5 4 4 5 4 4
52382024-02-27T21:46:04.625Z12/30 Flush
52392024-02-27T21:46:04.625Z13/30 Write block 534 len 40960 data: 3 3 3 3 3 3 3 4 4 3
52402024-02-27T21:46:04.625Z14/30 Flush
52412024-02-27T21:46:04.625Z15/30 Flush
52422024-02-27T21:46:04.626Z16/30 Write block 25 len 16384 data: 7 6 5 5
52432024-02-27T21:46:04.626Z17/30 Flush
52442024-02-27T21:46:04.626Z18/30 Write block 208 len 36864 data: 5 5 5 5 5 3 3 3 4
52452024-02-27T21:46:04.626Z19/30 Write block 511 len 40960 data: 7 7 6 6 5 6 6 4 4 4
52462024-02-27T21:46:04.626Z20/30 Read block 121 len 40960
52472024-02-27T21:46:04.627Z21/30 Flush
52482024-02-27T21:46:04.627Z22/30 Write block 484 len 16384 data: 8 8 8 7
52492024-02-27T21:46:04.627Z23/30 Read block 135 len 28672
52502024-02-27T21:46:04.627Z24/30 Read block 214 len 4096
52512024-02-27T21:46:04.627Z25/30 Read block 541 len 16384
52522024-02-27T21:46:04.627Z26/30 Read block 336 len 40960
52532024-02-27T21:46:04.628Z27/30 Read block 294 len 24576
52542024-02-27T21:46:04.628Z28/30 Write block 34 len 12288 data: 6 5 5
52552024-02-27T21:46:04.628Z29/30 Write block 109 len 16384 data: 7 6 7 5
52562024-02-27T21:46:04.628Z30/30 Write block 107 len 32768 data: 7 7 8 7 8 6 4 3
52572024-02-27T21:46:04.629Z----------------------------------------------------------------
52582024-02-27T21:46:04.629Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:29
52592024-02-27T21:46:04.629ZStates: Active Active Active
52602024-02-27T21:46:04.629ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52612024-02-27T21:46:04.629Z 2 Acked 1001 Write 3 Done Done Done false
52622024-02-27T21:46:04.629Z 3 Acked 1002 Write 3 Done Done Done false
52632024-02-27T21:46:04.630Z 4 Acked 1003 Write 1 Done Done Done false
52642024-02-27T21:46:04.630Z 5 Acked 1004 Read 2 Done Done Sent false
52652024-02-27T21:46:04.697Z 6 Acked 1005 Read 7 Done Done Sent false
52662024-02-27T21:46:04.697Z 7 Acked 1006 Read 4 Done Done Sent false
52672024-02-27T21:46:04.698Z 8 Acked 1007 Flush 0 Done Done Sent false
52682024-02-27T21:46:04.698Z 9 Acked 1008 Flush 0 Done Done Sent false
52692024-02-27T21:46:04.698Z 10 Acked 1009 Write 10 Done Done Sent false
52702024-02-27T21:46:04.698Z 11 Acked 1010 Write 8 Done Done Sent false
52712024-02-27T21:46:04.698Z 12 Acked 1011 Flush 0 Done Done Sent false
52722024-02-27T21:46:04.698Z 13 Acked 1012 Write 10 Done Done Sent false
52732024-02-27T21:46:04.698Z 14 Acked 1013 Flush 0 Done Done Sent false
52742024-02-27T21:46:04.698Z 15 Acked 1014 Flush 0 Done Done Sent false
52752024-02-27T21:46:04.699Z 16 Acked 1015 Write 4 Done Done Sent false
52762024-02-27T21:46:04.699Z 17 Acked 1016 Flush 0 Done Done Sent false
52772024-02-27T21:46:04.699Z 18 Acked 1017 Write 9 Done Done Sent false
52782024-02-27T21:46:04.719Z 19 Acked 1018 Write 10 Done Done Sent false
52792024-02-27T21:46:04.719Z 20 Acked 1019 Read 10 Done Done Sent false
52802024-02-27T21:46:04.720Z 21 Acked 1020 Flush 0 Done Done Sent false
52812024-02-27T21:46:04.720Z 22 Acked 1021 Write 4 Done Done Sent false
52822024-02-27T21:46:04.720Z 23 Acked 1022 Read 7 Done Done Sent false
52832024-02-27T21:46:04.720Z 24 Acked 1023 Read 1 Done Done Sent false
52842024-02-27T21:46:04.720Z 25 Acked 1024 Read 4 Done Done Sent false
52852024-02-27T21:46:04.720Z 26 Acked 1025 Read 10 Done Done Sent false
52862024-02-27T21:46:04.720Z 27 Acked 1026 Read 6 Done Done Sent false
52872024-02-27T21:46:04.721Z 28 Acked 1027 Write 3 Done Done Sent false
52882024-02-27T21:46:04.721Z 29 Acked 1028 Write 4 Done Done Sent false
52892024-02-27T21:46:04.721Z 30 Acked 1029 Write 8 Sent Sent Sent false
52902024-02-27T21:46:04.721Z STATES DS:0 DS:1 DS:2 TOTAL
52912024-02-27T21:46:04.721Z New 0 0 0 0
52922024-02-27T21:46:04.721Z Sent 1 1 26 28
52932024-02-27T21:46:04.721Z Done 28 28 3 59
52942024-02-27T21:46:04.722Z Skipped 0 0 0 0
52952024-02-27T21:46:04.722Z Error 0 0 0 0
52962024-02-27T21:46:04.722ZLast Flush: 1020 1020 1000
52972024-02-27T21:46:04.722ZDownstairs last five completed: 1000
52982024-02-27T21:46:04.722ZUpstairs last five completed: 30 29 28 27 26
52992024-02-27T21:46:04.722Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:02.182842697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1330}
53002024-02-27T21:46:04.723ZWrote out file "/tmp/test_repair_verify.data"
53012024-02-27T21:46:04.723Z
53022024-02-27T21:46:04.723ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53032024-02-27T21:46:04.723ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53042024-02-27T21:46:04.723Z 0 000-019 46 46 46 88 88 88 F F F
53052024-02-27T21:46:04.723Z 1 020-039 48 48 46 97 97 88 T T F <---
53062024-02-27T21:46:04.724Z 2 040-059 48 48 46 92 92 88 F F T <---
53072024-02-27T21:46:04.724Z 3 060-079 37 37 37 65 65 65 F F F
53082024-02-27T21:46:04.724Z 4 080-099 41 41 41 72 72 72 F F F
53092024-02-27T21:46:04.724Z 5 100-119 41 41 41 72 72 72 T T F <---
53102024-02-27T21:46:04.725Z 6 120-139 41 41 41 72 72 72 F F F
53112024-02-27T21:46:04.725Z 7 140-159 41 41 41 72 72 72 F F F
53122024-02-27T21:46:04.725Z 8 160-179 48 48 44 94 94 80 F F F <---
53132024-02-27T21:46:04.725Z 9 180-199 48 48 42 94 94 74 F F F <---
53142024-02-27T21:46:04.725Z 10 200-219 48 48 46 98 98 88 F F F <---
53152024-02-27T21:46:04.725Z 11 220-239 46 46 46 88 88 88 F F F
53162024-02-27T21:46:04.725Z 12 240-259 46 46 46 83 83 83 F F F
53172024-02-27T21:46:04.726Z 13 260-279 46 46 46 83 83 83 F F F
53182024-02-27T21:46:04.726Z 14 280-299 47 47 47 90 90 90 F F F
53192024-02-27T21:46:04.726Z 15 300-319 47 47 47 90 90 90 F F F
53202024-02-27T21:46:04.726Z 16 320-339 46 46 46 87 87 87 F F F
53212024-02-27T21:46:04.726Z 17 340-359 48 48 44 92 92 79 F F T <---
53222024-02-27T21:46:04.726Z 18 360-379 48 48 37 92 92 65 F F T <---
53232024-02-27T21:46:04.726Z 19 380-399 47 47 47 90 90 90 F F F
53242024-02-27T21:46:04.727Z 20 400-419 48 48 40 92 92 70 F F T <---
53252024-02-27T21:46:04.727Z 21 420-439 41 41 41 72 72 72 F F F
53262024-02-27T21:46:04.727Z 22 440-459 43 43 43 75 75 75 F F F
53272024-02-27T21:46:04.727Z 23 460-479 46 46 46 89 89 89 F F F
53282024-02-27T21:46:04.727Z 24 480-499 46 46 46 88 88 88 T T F <---
53292024-02-27T21:46:04.727Z 25 500-519 48 48 44 98 98 80 F F F <---
53302024-02-27T21:46:04.727Z 26 520-539 48 48 46 98 98 89 F F F <---
53312024-02-27T21:46:04.727Z 27 540-559 48 48 46 95 95 86 F F F <---
53322024-02-27T21:46:04.728Z 28 560-579 43 43 43 75 75 75 F F F
53332024-02-27T21:46:04.728Z 29 580-599 44 44 44 79 79 79 F F F
53342024-02-27T21:46:04.728ZMax gen: 48, Max flush: 98
53352024-02-27T21:46:04.728ZError: Difference in extent metadata found!
53362024-02-27T21:46:04.728ZA Difference in extent metadata is expected here
53372024-02-27T21:46:04.728ZOn loop 24
53382024-02-27T21:46:04.729Z
53392024-02-27T21:46:04.729ZVerifying data now
53402024-02-27T21:46:09.452ZNo change after verify
53412024-02-27T21:46:09.452ZLoop: 24 Downstairs dump after verify (and repair):
53422024-02-27T21:46:09.465ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53432024-02-27T21:46:09.465Z 0 000-019 46 46 46 88 88 88 F F F
53442024-02-27T21:46:09.465Z 1 020-039 49 49 49 99 99 99 F F F
53452024-02-27T21:46:09.465Z 2 040-059 48 48 48 92 92 92 F F F
53462024-02-27T21:46:09.466Z 3 060-079 37 37 37 65 65 65 F F F
53472024-02-27T21:46:09.466Z 4 080-099 41 41 41 72 72 72 F F F
53482024-02-27T21:46:09.479Z 5 100-119 49 49 49 99 99 99 F F F
53492024-02-27T21:46:09.479Z 6 120-139 41 41 41 72 72 72 F F F
53502024-02-27T21:46:09.480Z 7 140-159 41 41 41 72 72 72 F F F
53512024-02-27T21:46:09.480Z 8 160-179 48 48 48 94 94 94 F F F
53522024-02-27T21:46:09.480Z 9 180-199 48 48 48 94 94 94 F F F
53532024-02-27T21:46:09.480Z 10 200-219 48 48 48 98 98 98 F F F
53542024-02-27T21:46:09.480Z 11 220-239 46 46 46 88 88 88 F F F
53552024-02-27T21:46:09.480Z 12 240-259 46 46 46 83 83 83 F F F
53562024-02-27T21:46:09.480Z 13 260-279 46 46 46 83 83 83 F F F
53572024-02-27T21:46:09.481Z 14 280-299 47 47 47 90 90 90 F F F
53582024-02-27T21:46:09.481Z 15 300-319 47 47 47 90 90 90 F F F
53592024-02-27T21:46:09.481Z 16 320-339 46 46 46 87 87 87 F F F
53602024-02-27T21:46:09.481Z 17 340-359 48 48 48 92 92 92 F F F
53612024-02-27T21:46:09.481Z 18 360-379 48 48 48 92 92 92 F F F
53622024-02-27T21:46:09.481Z 19 380-399 47 47 47 90 90 90 F F F
53632024-02-27T21:46:09.481Z 20 400-419 48 48 48 92 92 92 F F F
53642024-02-27T21:46:09.481Z 21 420-439 41 41 41 72 72 72 F F F
53652024-02-27T21:46:09.481Z 22 440-459 43 43 43 75 75 75 F F F
53662024-02-27T21:46:09.481Z 23 460-479 46 46 46 89 89 89 F F F
53672024-02-27T21:46:09.482Z 24 480-499 49 49 49 99 99 99 F F F
53682024-02-27T21:46:09.482Z 25 500-519 48 48 48 98 98 98 F F F
53692024-02-27T21:46:09.482Z 26 520-539 48 48 48 98 98 98 F F F
53702024-02-27T21:46:09.482Z 27 540-559 48 48 48 95 95 95 F F F
53712024-02-27T21:46:09.482Z 28 560-579 43 43 43 75 75 75 F F F
53722024-02-27T21:46:09.482Z 29 580-599 44 44 44 79 79 79 F F F
53732024-02-27T21:46:09.482ZMax gen: 49, Max flush: 99
53742024-02-27T21:46:09.482Z
53752024-02-27T21:46:09.482ZBegin loop 25
53762024-02-27T21:46:09.483ZDownstairs to restart: 0
53772024-02-27T21:46:09.505Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886063971Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338}
53782024-02-27T21:46:09.505Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886351037Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338}
53792024-02-27T21:46:09.505Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886369605Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338}
53802024-02-27T21:46:09.505Z{"msg":"Crucible 48daebb6-4c1a-4415-bfa9-bacf36bfd3ff has session id: 75e17127-3174-44e8-a88b-1632b7e52003","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886390033Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53812024-02-27T21:46:09.505Z{"msg":"Crucible runtime is spawned
53822024-02-27T21:46:09.505ZUpstairs opts: Upstairs UUID: 48daebb6-4c1a-4415-bfa9-bacf36bfd3ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886443586Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53832024-02-27T21:46:09.505Z{"msg":"Crucible stats registered with UUID: 48daebb6-4c1a-4415-bfa9-bacf36bfd3ff","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886468593Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53842024-02-27T21:46:09.505Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.88649903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338}
53852024-02-27T21:46:09.506Z{"msg":"48daebb6-4c1a-4415-bfa9-bacf36bfd3ff active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886557113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53862024-02-27T21:46:09.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886607827Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53872024-02-27T21:46:09.506Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886702796Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53882024-02-27T21:46:09.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886797575Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53892024-02-27T21:46:09.506Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886850359Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53902024-02-27T21:46:09.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886892583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53912024-02-27T21:46:09.506Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.88692341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53922024-02-27T21:46:09.506Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886949687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53932024-02-27T21:46:09.506Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.886987892Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"io task","client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53942024-02-27T21:46:09.506Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.887033637Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"io task","client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53952024-02-27T21:46:09.506Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.887059974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53962024-02-27T21:46:09.506Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.88709473Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"io task","client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53972024-02-27T21:46:09.506Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.887112688Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"io task","client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53982024-02-27T21:46:09.506Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.8878575Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
53992024-02-27T21:46:09.506Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.887912933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54002024-02-27T21:46:09.506Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.88794599Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54012024-02-27T21:46:09.506Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.887996933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54022024-02-27T21:46:09.507Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.88844848Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54032024-02-27T21:46:09.507Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.888544429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54042024-02-27T21:46:09.507Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.888595063Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54052024-02-27T21:46:09.507Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.888636048Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54062024-02-27T21:46:09.507Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889137669Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54072024-02-27T21:46:09.507Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.8893017Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54082024-02-27T21:46:09.507Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889335616Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54092024-02-27T21:46:09.507Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889361133Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54102024-02-27T21:46:09.507Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889388389Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54112024-02-27T21:46:09.507Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889413077Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54122024-02-27T21:46:09.507Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889439214Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54132024-02-27T21:46:09.507Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889584177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54142024-02-27T21:46:09.507Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889620882Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54152024-02-27T21:46:09.507Z{"msg":"[0]R flush_numbers[0..12]: [88, 99, 92, 65, 72, 99, 72, 72, 94, 94, 98, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.88964085Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54162024-02-27T21:46:09.507Z{"msg":"[0]R generation[0..12]: [46, 49, 48, 37, 41, 49, 41, 41, 48, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889671016Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54172024-02-27T21:46:09.508Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889703952Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54182024-02-27T21:46:09.508Z{"msg":"[1]R flush_numbers[0..12]: [88, 99, 92, 65, 72, 99, 72, 72, 94, 94, 98, 88]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
54192024-02-27T21:46:09.508Z2024-02-27T21:46:07.889740808Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54202024-02-27T21:46:09.508Z{"msg":"[1]R generation[0..12]: [46, 49, 48, 37, 41, 49, 41, 41, 48, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889779163Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54212024-02-27T21:46:09.508Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.88980634Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54222024-02-27T21:46:09.508Z{"msg":"[2]R flush_numbers[0..12]: [88, 99, 92, 65, 72, 99, 72, 72, 94, 94, 98, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889837277Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54232024-02-27T21:46:09.508Z{"msg":"[2]R generation[0..12]: [46, 49, 48, 37, 41, 49, 41, 41, 48, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889862234Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54242024-02-27T21:46:09.508Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.88989142Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54252024-02-27T21:46:09.508Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889915957Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54262024-02-27T21:46:09.508Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889940255Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54272024-02-27T21:46:09.509Z{"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889970231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54282024-02-27T21:46:09.509Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.889994468Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54292024-02-27T21:46:09.509Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.890022645Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54302024-02-27T21:46:09.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.890044902Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54312024-02-27T21:46:09.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.890072339Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54322024-02-27T21:46:09.509Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.890096116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54332024-02-27T21:46:09.509Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.890122733Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54342024-02-27T21:46:09.509Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54352024-02-27T21:46:09.509Z"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.890154309Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54362024-02-27T21:46:09.509Z{"msg":"48daebb6-4c1a-4415-bfa9-bacf36bfd3ff is now active with session: 75e17127-3174-44e8-a88b-1632b7e52003","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.890183636Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54372024-02-27T21:46:09.509Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.890205663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338,"session_id":"75e17127-3174-44e8-a88b-1632b7e52003"}
54382024-02-27T21:46:09.509Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:07.89023158Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338}
54392024-02-27T21:46:09.525ZLoading write count information from file "/tmp/test_repair_verify.data"
54402024-02-27T21:46:09.525ZRun Repair workload
54412024-02-27T21:46:09.526Z01/30 Write block 9 len 36864 data: 6 6 6 6 5 7 6 6 5
54422024-02-27T21:46:09.526Z02/30 Read block 485 len 32768
54432024-02-27T21:46:09.526Z03/30 Read block 0 len 32768
54442024-02-27T21:46:09.526Z04/30 Read block 317 len 40960
54452024-02-27T21:46:09.526Z05/30 Read block 420 len 8192
54462024-02-27T21:46:09.526Z06/30 Flush
54472024-02-27T21:46:09.526Z07/30 Write block 536 len 4096 data: 4
54482024-02-27T21:46:09.527Z08/30 Read block 211 len 40960
54492024-02-27T21:46:09.527Z09/30 Write block 496 len 16384 data: 5 5 7 7
54502024-02-27T21:46:09.527Z10/30 Write block 236 len 36864 data: 7 4 3 3 3 2 2 2 3
54512024-02-27T21:46:09.527Z11/30 Write block 24 len 4096 data: 7
54522024-02-27T21:46:09.527Z12/30 Read block 514 len 24576
54532024-02-27T21:46:09.527Z13/30 Read block 325 len 4096
54542024-02-27T21:46:09.527Z14/30 Write block 88 len 16384 data: 6 7 5 4
54552024-02-27T21:46:09.527Z15/30 Read block 237 len 12288
54562024-02-27T21:46:09.527Z16/30 Read block 430 len 12288
54572024-02-27T21:46:09.527Z17/30 Read block 57 len 40960
54582024-02-27T21:46:09.528Z18/30 Read block 21 len 36864
54592024-02-27T21:46:09.528Z19/30 Flush
54602024-02-27T21:46:09.528Z20/30 Write block 191 len 40960 data: 7 6 5 5 4 4 4 4 4 6
54612024-02-27T21:46:09.528Z21/30 Write block 416 len 40960 data: 4 6 6 6 3 2 2 2 2 2
54622024-02-27T21:46:09.528Z22/30 Read block 375 len 12288
54632024-02-27T21:46:09.528Z23/30 Read block 116 len 8192
54642024-02-27T21:46:09.528Z24/30 Read block 275 len 12288
54652024-02-27T21:46:09.528Z25/30 Read block 529 len 40960
54662024-02-27T21:46:09.560Z26/30 Flush
54672024-02-27T21:46:09.560Z27/30 Write block 466 len 24576 data: 7 6 4 3 2 2
54682024-02-27T21:46:09.560Z28/30 Write block 382 len 20480 data: 2 3 4 6 6
54692024-02-27T21:46:09.560Z29/30 Read block 287 len 40960
54702024-02-27T21:46:09.560Z30/30 Write block 531 len 28672 data: 3 2 3 4 4 5 4
54712024-02-27T21:46:09.561Z----------------------------------------------------------------
54722024-02-27T21:46:09.561Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
54732024-02-27T21:46:09.561ZStates: Active Active Active
54742024-02-27T21:46:09.561ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54752024-02-27T21:46:09.561Z 1 Acked 1000 Write 9 Sent Done Done false
54762024-02-27T21:46:09.561Z 2 Acked 1001 Read 8 Sent Done Done false
54772024-02-27T21:46:09.561Z 3 Acked 1002 Read 8 Sent Done Done false
54782024-02-27T21:46:09.561Z 4 Acked 1003 Read 10 Sent Done Done false
54792024-02-27T21:46:09.561Z 5 Acked 1004 Read 2 Sent Done Done false
54802024-02-27T21:46:09.561Z 6 Acked 1005 Flush 0 Sent Done Done false
54812024-02-27T21:46:09.561Z 7 Acked 1006 Write 1 Sent Done Done false
54822024-02-27T21:46:09.561Z 8 Acked 1007 Read 10 Sent Done Done false
54832024-02-27T21:46:09.562Z 9 Acked 1008 Write 4 Sent Done Done false
54842024-02-27T21:46:09.562Z 10 Acked 1009 Write 9 Sent Done Done false
54852024-02-27T21:46:09.562Z 11 Acked 1010 Write 1 Sent Done Done false
54862024-02-27T21:46:09.562Z 12 Acked 1011 Read 6 Sent Done Done false
54872024-02-27T21:46:09.562Z 13 Acked 1012 Read 1 Sent Done Done false
54882024-02-27T21:46:09.562Z 14 Acked 1013 Write 4 Sent Done Done false
54892024-02-27T21:46:09.562Z 15 Acked 1014 Read 3 Sent Done Done false
54902024-02-27T21:46:09.562Z 16 Acked 1015 Read 3 Sent Done Done false
54912024-02-27T21:46:09.562Z 17 Acked 1016 Read 10 Sent Done Done false
54922024-02-27T21:46:09.562Z 18 Acked 1017 Read 9 Sent Done Done false
54932024-02-27T21:46:09.563Z 19 Acked 1018 Flush 0 Sent Done Done false
54942024-02-27T21:46:09.563Z 20 Acked 1019 Write 10 Sent Done Done false
54952024-02-27T21:46:09.563Z 21 Acked 1020 Write 10 Sent Done Done false
54962024-02-27T21:46:09.563Z 22 Acked 1021 Read 3 Sent Done Done false
54972024-02-27T21:46:09.563Z 23 Acked 1022 Read 2 Sent Done Done false
54982024-02-27T21:46:09.563Z 24 Acked 1023 Read 3 Sent Done Done false
54992024-02-27T21:46:09.563Z 25 Acked 1024 Read 10 Sent Done Done false
55002024-02-27T21:46:09.563Z 26 Acked 1025 Flush 0 Sent Done Done false
55012024-02-27T21:46:09.563Z 27 Acked 1026 Write 6 Sent Done Done false
55022024-02-27T21:46:09.563Z 28 Acked 1027 Write 5 Sent Done Done false
55032024-02-27T21:46:09.564Z 29 Acked 1028 Read 10 Sent Done Done false
55042024-02-27T21:46:09.564Z 30 Acked 1029 Write 7 Sent Sent Sent false
55052024-02-27T21:46:09.564Z STATES DS:0 DS:1 DS:2 TOTAL
55062024-02-27T21:46:09.564Z New 0 0 0 0
55072024-02-27T21:46:09.564Z Sent 30 1 1 32
55082024-02-27T21:46:09.564Z Done 0 29 29 58
55092024-02-27T21:46:09.564Z Skipped 0 0 0 0
55102024-02-27T21:46:09.564Z Error 0 0 0 0
55112024-02-27T21:46:09.564ZLast Flush: 0 1025 1025
55122024-02-27T21:46:09.564ZDownstairs last five completed:
55132024-02-27T21:46:09.564ZUpstairs last five completed: 30 29 28 27 26
55142024-02-27T21:46:09.565Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:07.923433939Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1338}
55152024-02-27T21:46:09.565ZWrote out file "/tmp/test_repair_verify.data"
55162024-02-27T21:46:09.565Z
55172024-02-27T21:46:09.565ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55182024-02-27T21:46:09.592ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55192024-02-27T21:46:09.593Z 0 000-019 46 50 50 88 100 100 F F F <---
55202024-02-27T21:46:09.593Z 1 020-039 49 50 50 99 101 101 F F F <---
55212024-02-27T21:46:09.593Z 2 040-059 48 48 48 92 92 92 F F F
55222024-02-27T21:46:09.593Z 3 060-079 37 37 37 65 65 65 F F F
55232024-02-27T21:46:09.593Z 4 080-099 41 50 50 72 101 101 F F F <---
55242024-02-27T21:46:09.593Z 5 100-119 49 49 49 99 99 99 F F F
55252024-02-27T21:46:09.593Z 6 120-139 41 41 41 72 72 72 F F F
55262024-02-27T21:46:09.593Z 7 140-159 41 41 41 72 72 72 F F F
55272024-02-27T21:46:09.594Z 8 160-179 48 48 48 94 94 94 F F F
55282024-02-27T21:46:09.594Z 9 180-199 48 50 50 94 102 102 F F F <---
55292024-02-27T21:46:09.594Z 10 200-219 48 50 50 98 102 102 F F F <---
55302024-02-27T21:46:09.594Z 11 220-239 46 50 50 88 101 101 F F F <---
55312024-02-27T21:46:09.594Z 12 240-259 46 50 50 83 101 101 F F F <---
55322024-02-27T21:46:09.594Z 13 260-279 46 46 46 83 83 83 F F F
55332024-02-27T21:46:09.594Z 14 280-299 47 47 47 90 90 90 F F F
55342024-02-27T21:46:09.594Z 15 300-319 47 47 47 90 90 90 F F F
55352024-02-27T21:46:09.594Z 16 320-339 46 46 46 87 87 87 F F F
55362024-02-27T21:46:09.594Z 17 340-359 48 48 48 92 92 92 F F F
55372024-02-27T21:46:09.594Z 18 360-379 48 48 48 92 92 92 F F F
55382024-02-27T21:46:09.595Z 19 380-399 47 47 47 90 90 90 F T T <---
55392024-02-27T21:46:09.595Z 20 400-419 48 50 50 92 102 102 F F F <---
55402024-02-27T21:46:09.595Z 21 420-439 41 50 50 72 102 102 F F F <---
55412024-02-27T21:46:09.595Z 22 440-459 43 43 43 75 75 75 F F F
55422024-02-27T21:46:09.595Z 23 460-479 46 46 46 89 89 89 F T T <---
55432024-02-27T21:46:09.595Z 24 480-499 49 50 50 99 101 101 F F F <---
55442024-02-27T21:46:09.595Z 25 500-519 48 48 48 98 98 98 F F F
55452024-02-27T21:46:09.595Z 26 520-539 48 50 50 98 101 101 F T T <---
55462024-02-27T21:46:09.595Z 27 540-559 48 48 48 95 95 95 F F F
55472024-02-27T21:46:09.595Z 28 560-579 43 43 43 75 75 75 F F F
55482024-02-27T21:46:09.596Z 29 580-599 44 44 44 79 79 79 F F F
55492024-02-27T21:46:09.596ZMax gen: 50, Max flush: 102
55502024-02-27T21:46:09.596ZError: Difference in extent metadata found!
55512024-02-27T21:46:09.596ZA Difference in extent metadata is expected here
55522024-02-27T21:46:09.596ZOn loop 25
55532024-02-27T21:46:09.596Z
55542024-02-27T21:46:09.596ZVerifying data now
55552024-02-27T21:46:14.196ZNo change after verify
55562024-02-27T21:46:14.197ZLoop: 25 Downstairs dump after verify (and repair):
55572024-02-27T21:46:14.211ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55582024-02-27T21:46:14.211Z 0 000-019 50 50 50 100 100 100 F F F
55592024-02-27T21:46:14.211Z 1 020-039 50 50 50 101 101 101 F F F
55602024-02-27T21:46:14.211Z 2 040-059 48 48 48 92 92 92 F F F
55612024-02-27T21:46:14.212Z 3 060-079 37 37 37 65 65 65 F F F
55622024-02-27T21:46:14.212Z 4 080-099 50 50 50 101 101 101 F F F
55632024-02-27T21:46:14.212Z 5 100-119 49 49 49 99 99 99 F F F
55642024-02-27T21:46:14.212Z 6 120-139 41 41 41 72 72 72 F F F
55652024-02-27T21:46:14.216Z 7 140-159 41 41 41 72 72 72 F F F
55662024-02-27T21:46:14.216Z 8 160-179 48 48 48 94 94 94 F F F
55672024-02-27T21:46:14.216Z 9 180-199 50 50 50 102 102 102 F F F
55682024-02-27T21:46:14.216Z 10 200-219 50 50 50 102 102 102 F F F
55692024-02-27T21:46:14.216Z 11 220-239 50 50 50 101 101 101 F F F
55702024-02-27T21:46:14.217Z 12 240-259 50 50 50 101 101 101 F F F
55712024-02-27T21:46:14.217Z 13 260-279 46 46 46 83 83 83 F F F
55722024-02-27T21:46:14.217Z 14 280-299 47 47 47 90 90 90 F F F
55732024-02-27T21:46:14.217Z 15 300-319 47 47 47 90 90 90 F F F
55742024-02-27T21:46:14.217Z 16 320-339 46 46 46 87 87 87 F F F
55752024-02-27T21:46:14.217Z 17 340-359 48 48 48 92 92 92 F F F
55762024-02-27T21:46:14.217Z 18 360-379 48 48 48 92 92 92 F F F
55772024-02-27T21:46:14.217Z 19 380-399 51 51 51 103 103 103 F F F
55782024-02-27T21:46:14.217Z 20 400-419 50 50 50 102 102 102 F F F
55792024-02-27T21:46:14.218Z 21 420-439 50 50 50 102 102 102 F F F
55802024-02-27T21:46:14.218Z 22 440-459 43 43 43 75 75 75 F F F
55812024-02-27T21:46:14.218Z 23 460-479 51 51 51 103 103 103 F F F
55822024-02-27T21:46:14.218Z 24 480-499 50 50 50 101 101 101 F F F
55832024-02-27T21:46:14.218Z 25 500-519 48 48 48 98 98 98 F F F
55842024-02-27T21:46:14.218Z 26 520-539 51 51 51 103 103 103 F F F
55852024-02-27T21:46:14.218Z 27 540-559 48 48 48 95 95 95 F F F
55862024-02-27T21:46:14.218Z 28 560-579 43 43 43 75 75 75 F F F
55872024-02-27T21:46:14.218Z 29 580-599 44 44 44 79 79 79 F F F
55882024-02-27T21:46:14.218ZMax gen: 51, Max flush: 103
55892024-02-27T21:46:14.218Z
55902024-02-27T21:46:14.219ZBegin loop 26
55912024-02-27T21:46:14.219ZDownstairs to restart: 1
55922024-02-27T21:46:14.238Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.632456822Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346}
55932024-02-27T21:46:14.248Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.632792063Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346}
55942024-02-27T21:46:14.248Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.632887401Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346}
55952024-02-27T21:46:14.248Z{"msg":"Crucible dc6a434c-8a21-4ac6-896c-fbcb7ad6f591 has session id: 68645428-fc94-4409-947a-4cf3bce54ef0","v":0,"name":"crucible","level":30Crucible runtime is spawned
55962024-02-27T21:46:14.248Z,"time":"2024-02-27T21:46:12.632944455Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
55972024-02-27T21:46:14.248Z{"msg":"Upstairs opts: Upstairs UUID: dc6a434c-8a21-4ac6-896c-fbcb7ad6f591, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.632991279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
55982024-02-27T21:46:14.249Z{"msg":"Crucible stats registered with UUID: dc6a434c-8a21-4ac6-896c-fbcb7ad6f591","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633054282Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
55992024-02-27T21:46:14.249Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633089148Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346}
56002024-02-27T21:46:14.249Z{"msg":"dc6a434c-8a21-4ac6-896c-fbcb7ad6f591 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633120604Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56012024-02-27T21:46:14.249Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.6331565Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56022024-02-27T21:46:14.249Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633196855Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56032024-02-27T21:46:14.249Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633222222Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56042024-02-27T21:46:14.249Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633258168Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56052024-02-27T21:46:14.249Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633284165Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56062024-02-27T21:46:14.249Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.63332196Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56072024-02-27T21:46:14.249Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633345038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"io task","client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56082024-02-27T21:46:14.249Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633376494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"io task","client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56092024-02-27T21:46:14.249Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633395332Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56102024-02-27T21:46:14.249Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.63357582Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"io task","client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56112024-02-27T21:46:14.249Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633611956Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56122024-02-27T21:46:14.249Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.633628374Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"io task","client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56132024-02-27T21:46:14.249Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.63434467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56142024-02-27T21:46:14.249Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.634387425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56152024-02-27T21:46:14.250Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.634403123Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56162024-02-27T21:46:14.250Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.634418051Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56172024-02-27T21:46:14.250Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.634447548Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56182024-02-27T21:46:14.250Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.634469295Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56192024-02-27T21:46:14.250Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.634997123Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56202024-02-27T21:46:14.250Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.63510403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56212024-02-27T21:46:14.250Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635242384Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56222024-02-27T21:46:14.250Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635638377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56232024-02-27T21:46:14.250Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635696331Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56242024-02-27T21:46:14.250Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635734286Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56252024-02-27T21:46:14.250Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635770212Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56262024-02-27T21:46:14.250Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.63578695Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56272024-02-27T21:46:14.250Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635809467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56282024-02-27T21:46:14.250Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635822366Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56292024-02-27T21:46:14.250Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635843753Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56302024-02-27T21:46:14.250Z{"msg":"[0]R flush_numbers[0..12]: [100, 101, 92, 65, 101, 99, 72, 72, 94, 102, 102, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635859851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56312024-02-27T21:46:14.250Z{"msg":"[0]R generation[0..12]: [50, 50, 48, 37, 50, 49, 41, 41, 48, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635881089Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56322024-02-27T21:46:14.250Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635896617Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56332024-02-27T21:46:14.251Z{"msg":"[1]R flush_numbers[0..12]: [100, 101, 92, 65, 101, 99, 72, 72, 94, 102, 102, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635917295Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56342024-02-27T21:46:14.251Z{"msg":"[1]R generation[0..12]: [50, 50, 48, 37, 50, 49, 41, 41, 48, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635932213Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56352024-02-27T21:46:14.251Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.63595393Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56362024-02-27T21:46:14.251Z{"msg":"[2]R flush_numbers[0..12]: [100, 101, 92, 65, 101, 99, 72, 72, 94, 102, 102, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635968429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56372024-02-27T21:46:14.251Z{"msg":"[2]R generation[0..12]: [50, 50, 48, 37, 50, 49, 41, 41, 48, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635983037Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56382024-02-27T21:46:14.251Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.635997755Z","hostname":"Wait for a query_work_queue command to finish before sending IO
56392024-02-27T21:46:14.251Zip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56402024-02-27T21:46:14.251Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.63603625Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56412024-02-27T21:46:14.251Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636069177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56422024-02-27T21:46:14.251Z{"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636089664Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56432024-02-27T21:46:14.251Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636118441Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56442024-02-27T21:46:14.251Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636136649Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56452024-02-27T21:46:14.251Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636166355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56462024-02-27T21:46:14.251Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636183603Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56472024-02-27T21:46:14.251Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.63621361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56482024-02-27T21:46:14.251Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636234687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56492024-02-27T21:46:14.251Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636262144Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56502024-02-27T21:46:14.252Z{"msg":"dc6a434c-8a21-4ac6-896c-fbcb7ad6f591 is now active with session: 68645428-fc94-4409-947a-4cf3bce54ef0","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636280982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56512024-02-27T21:46:14.252Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636308289Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346,"session_id":"68645428-fc94-4409-947a-4cf3bce54ef0"}
56522024-02-27T21:46:14.252Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:12.636326287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346}
56532024-02-27T21:46:14.252ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56542024-02-27T21:46:14.252ZLoading write count information from file "/tmp/test_repair_verify.data"
56552024-02-27T21:46:14.252ZRun Repair workload
56562024-02-27T21:46:14.252Z01/30 Write block 573 len 24576 data: 6 4 5 5 5 6
56572024-02-27T21:46:14.252Z02/30 Read block 178 len 12288
56582024-02-27T21:46:14.252Z03/30 Read block 353 len 12288
56592024-02-27T21:46:14.252Z04/30 Read block 281 len 4096
56602024-02-27T21:46:14.252Z05/30 Read block 566 len 4096
56612024-02-27T21:46:14.252Z06/30 Write block 509 len 20480 data: 5 6 8 8 7
56622024-02-27T21:46:14.252Z07/30 Read block 264 len 32768
56632024-02-27T21:46:14.252Z08/30 Read block 355 len 16384
56642024-02-27T21:46:14.252Z09/30 Read block 503 len 8192
56652024-02-27T21:46:14.252Z10/30 Read block 384 len 28672
56662024-02-27T21:46:14.256Z11/30 Read block 271 len 4096
56672024-02-27T21:46:14.256Z12/30 Write block 12 len 32768 data: 7 6 8 7 7 6 4 3
56682024-02-27T21:46:14.257Z13/30 Read block 89 len 4096
56692024-02-27T21:46:14.257Z14/30 Flush
56702024-02-27T21:46:14.257Z15/30 Write block 338 len 8192 data: 7 7
56712024-02-27T21:46:14.257Z16/30 Flush
56722024-02-27T21:46:14.257Z17/30 Write block 130 len 4096 data: 4
56732024-02-27T21:46:14.257Z18/30 Read block 468 len 28672
56742024-02-27T21:46:14.257Z19/30 Read block 566 len 28672
56752024-02-27T21:46:14.257Z20/30 Read block 223 len 28672
56762024-02-27T21:46:14.257Z21/30 Write block 217 len 12288 data: 7 8 7
56772024-02-27T21:46:14.257Z22/30 Read block 19 len 36864
56782024-02-27T21:46:14.262Z23/30 Read block 407 len 24576
56792024-02-27T21:46:14.262Z24/30 Read block 76 len 36864
56802024-02-27T21:46:14.262Z25/30 Write block 243 len 28672 data: 3 4 4 4 6 5 5
56812024-02-27T21:46:14.263Z26/30 Write block 399 len 12288 data: 4 5 6
56822024-02-27T21:46:14.263Z27/30 Read block 411 len 28672
56832024-02-27T21:46:14.263Z28/30 Read block 95 len 24576
56842024-02-27T21:46:14.267Z29/30 Read block 166 len 40960
56852024-02-27T21:46:14.267Z30/30 Read block 512 len 20480
56862024-02-27T21:46:14.267Z----------------------------------------------------------------
56872024-02-27T21:46:14.267Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
56882024-02-27T21:46:14.268ZStates: Active Active Active
56892024-02-27T21:46:14.268ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
56902024-02-27T21:46:14.268Z 1 Acked 1000 Write 6 Done Done Done false
56912024-02-27T21:46:14.268Z 2 Acked 1001 Read 3 Done Done Done false
56922024-02-27T21:46:14.268Z 3 Acked 1002 Read 3 Done Done Done false
56932024-02-27T21:46:14.268Z 4 Acked 1003 Read 1 Done Done Done false
56942024-02-27T21:46:14.268Z 5 Acked 1004 Read 1 Done Done Done false
56952024-02-27T21:46:14.268Z 6 Acked 1005 Write 5 Done Done Done false
56962024-02-27T21:46:14.269Z 7 Acked 1006 Read 8 Done Sent Done false
56972024-02-27T21:46:14.269Z 8 Acked 1007 Read 4 Done Sent Done false
56982024-02-27T21:46:14.269Z 9 Acked 1008 Read 2 Done Sent Done false
56992024-02-27T21:46:14.269Z 10 Acked 1009 Read 7 Done Sent Done false
57002024-02-27T21:46:14.269Z 11 Acked 1010 Read 1 Done Sent Done false
57012024-02-27T21:46:14.269Z 12 Acked 1011 Write 8 Done Sent Done false
57022024-02-27T21:46:14.269Z 13 Acked 1012 Read 1 Done Sent Done false
57032024-02-27T21:46:14.269Z 14 Acked 1013 Flush 0 Done Sent Done false
57042024-02-27T21:46:14.269Z 15 Acked 1014 Write 2 Done Sent Done false
57052024-02-27T21:46:14.269Z 16 Acked 1015 Flush 0 Done Sent Done false
57062024-02-27T21:46:14.269Z 17 Acked 1016 Write 1 Done Sent Done false
57072024-02-27T21:46:14.270Z 18 Acked 1017 Read 7 Done Sent Done false
57082024-02-27T21:46:14.270Z 19 Acked 1018 Read 7 Done Sent Done false
57092024-02-27T21:46:14.270Z 20 Acked 1019 Read 7 Done Sent Done false
57102024-02-27T21:46:14.270Z 21 Acked 1020 Write 3 Done Sent Done false
57112024-02-27T21:46:14.270Z 22 Acked 1021 Read 9 Done Sent Done false
57122024-02-27T21:46:14.270Z 23 Acked 1022 Read 6 Done Sent Done false
57132024-02-27T21:46:14.270Z 24 Acked 1023 Read 9 Done Sent Done false
57142024-02-27T21:46:14.270Z 25 Acked 1024 Write 7 Done Sent Done false
57152024-02-27T21:46:14.270Z 26 Acked 1025 Write 3 Done Sent Done false
57162024-02-27T21:46:14.270Z 27 Acked 1026 Read 7 Done Sent Done false
57172024-02-27T21:46:14.270Z 28 Acked 1027 Read 6 Done Sent Done false
57182024-02-27T21:46:14.271Z 29 Acked 1028 Read 10 Done Sent Done false
57192024-02-27T21:46:14.271Z 30 Acked 1029 Read 5 Done Sent Done false
57202024-02-27T21:46:14.271Z STATES DS:0 DS:1 DS:2 TOTAL
57212024-02-27T21:46:14.271Z New 0 0 0 0
57222024-02-27T21:46:14.271Z Sent 0 24 0 24
57232024-02-27T21:46:14.271Z Done 30 6 30 66
57242024-02-27T21:46:14.271Z Skipped 0 0 0 0
57252024-02-27T21:46:14.271Z Error 0 0 0 0
57262024-02-27T21:46:14.271ZLast Flush: 1015 0 1015
57272024-02-27T21:46:14.271ZDownstairs last five completed:
57282024-02-27T21:46:14.271ZUpstairs last five completed: 30 29 28 27 26
57292024-02-27T21:46:14.271Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:12.664721382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1346}
57302024-02-27T21:46:14.271ZWrote out file "/tmp/test_repair_verify.data"
57312024-02-27T21:46:14.276Z
57322024-02-27T21:46:14.277ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57332024-02-27T21:46:14.288ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57342024-02-27T21:46:14.288Z 0 000-019 52 50 52 104 100 104 F F F <---
57352024-02-27T21:46:14.288Z 1 020-039 50 50 50 101 101 101 F F F
57362024-02-27T21:46:14.288Z 2 040-059 48 48 48 92 92 92 F F F
57372024-02-27T21:46:14.288Z 3 060-079 37 37 37 65 65 65 F F F
57382024-02-27T21:46:14.288Z 4 080-099 50 50 50 101 101 101 F F F
57392024-02-27T21:46:14.288Z 5 100-119 49 49 49 99 99 99 F F F
57402024-02-27T21:46:14.292Z 6 120-139 41 41 41 72 72 72 T F T <---
57412024-02-27T21:46:14.292Z 7 140-159 41 41 41 72 72 72 F F F
57422024-02-27T21:46:14.292Z 8 160-179 48 48 48 94 94 94 F F F
57432024-02-27T21:46:14.292Z 9 180-199 50 50 50 102 102 102 F F F
57442024-02-27T21:46:14.292Z 10 200-219 50 50 50 102 102 102 T F T <---
57452024-02-27T21:46:14.292Z 11 220-239 50 50 50 101 101 101 F F F
57462024-02-27T21:46:14.292Z 12 240-259 50 50 50 101 101 101 T F T <---
57472024-02-27T21:46:14.292Z 13 260-279 46 46 46 83 83 83 F F F
57482024-02-27T21:46:14.292Z 14 280-299 47 47 47 90 90 90 F F F
57492024-02-27T21:46:14.292Z 15 300-319 47 47 47 90 90 90 F F F
57502024-02-27T21:46:14.292Z 16 320-339 52 46 52 105 87 105 F F F <---
57512024-02-27T21:46:14.292Z 17 340-359 48 48 48 92 92 92 F F F
57522024-02-27T21:46:14.292Z 18 360-379 48 48 48 92 92 92 F F F
57532024-02-27T21:46:14.292Z 19 380-399 51 51 51 103 103 103 T F T <---
57542024-02-27T21:46:14.292Z 20 400-419 50 50 50 102 102 102 T F T <---
57552024-02-27T21:46:14.292Z 21 420-439 50 50 50 102 102 102 F F F
57562024-02-27T21:46:14.292Z 22 440-459 43 43 43 75 75 75 F F F
57572024-02-27T21:46:14.293Z 23 460-479 51 51 51 103 103 103 F F F
57582024-02-27T21:46:14.293Z 24 480-499 50 50 50 101 101 101 F F F
57592024-02-27T21:46:14.293Z 25 500-519 52 48 52 104 98 104 F T F <---
57602024-02-27T21:46:14.293Z 26 520-539 51 51 51 103 103 103 F F F
57612024-02-27T21:46:14.293Z 27 540-559 48 48 48 95 95 95 F F F
57622024-02-27T21:46:14.293Z 28 560-579 52 43 52 104 75 104 F T F <---
57632024-02-27T21:46:14.293Z 29 580-599 44 44 44 79 79 79 F F F
57642024-02-27T21:46:14.293ZMax gen: 52, Max flush: 105
57652024-02-27T21:46:14.293ZError: Difference in extent metadata found!
57662024-02-27T21:46:14.293ZA Difference in extent metadata is expected here
57672024-02-27T21:46:14.293ZOn loop 26
57682024-02-27T21:46:14.293Z
57692024-02-27T21:46:14.296ZVerifying data now
57702024-02-27T21:46:18.772ZNo change after verify
57712024-02-27T21:46:18.772ZLoop: 26 Downstairs dump after verify (and repair):
57722024-02-27T21:46:18.781ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57732024-02-27T21:46:18.781Z 0 000-019 52 52 52 104 104 104 F F F
57742024-02-27T21:46:18.789Z 1 020-039 50 50 50 101 101 101 F F F
57752024-02-27T21:46:18.789Z 2 040-059 48 48 48 92 92 92 F F F
57762024-02-27T21:46:18.789Z 3 060-079 37 37 37 65 65 65 F F F
57772024-02-27T21:46:18.789Z 4 080-099 50 50 50 101 101 101 F F F
57782024-02-27T21:46:18.790Z 5 100-119 49 49 49 99 99 99 F F F
57792024-02-27T21:46:18.790Z 6 120-139 53 53 53 106 106 106 F F F
57802024-02-27T21:46:18.790Z 7 140-159 41 41 41 72 72 72 F F F
57812024-02-27T21:46:18.790Z 8 160-179 48 48 48 94 94 94 F F F
57822024-02-27T21:46:18.790Z 9 180-199 50 50 50 102 102 102 F F F
57832024-02-27T21:46:18.790Z 10 200-219 53 53 53 106 106 106 F F F
57842024-02-27T21:46:18.791Z 11 220-239 50 50 50 101 101 101 F F F
57852024-02-27T21:46:18.791Z 12 240-259 53 53 53 106 106 106 F F F
57862024-02-27T21:46:18.791Z 13 260-279 46 46 46 83 83 83 F F F
57872024-02-27T21:46:18.791Z 14 280-299 47 47 47 90 90 90 F F F
57882024-02-27T21:46:18.791Z 15 300-319 47 47 47 90 90 90 F F F
57892024-02-27T21:46:18.792Z 16 320-339 52 52 52 105 105 105 F F F
57902024-02-27T21:46:18.792Z 17 340-359 48 48 48 92 92 92 F F F
57912024-02-27T21:46:18.792Z 18 360-379 48 48 48 92 92 92 F F F
57922024-02-27T21:46:18.792Z 19 380-399 53 53 53 106 106 106 F F F
57932024-02-27T21:46:18.792Z 20 400-419 53 53 53 106 106 106 F F F
57942024-02-27T21:46:18.793Z 21 420-439 50 50 50 102 102 102 F F F
57952024-02-27T21:46:18.793Z 22 440-459 43 43 43 75 75 75 F F F
57962024-02-27T21:46:18.793Z 23 460-479 51 51 51 103 103 103 F F F
57972024-02-27T21:46:18.793Z 24 480-499 50 50 50 101 101 101 F F F
57982024-02-27T21:46:18.793Z 25 500-519 52 52 52 104 104 104 F F F
57992024-02-27T21:46:18.794Z 26 520-539 51 51 51 103 103 103 F F F
58002024-02-27T21:46:18.794Z 27 540-559 48 48 48 95 95 95 F F F
58012024-02-27T21:46:18.794Z 28 560-579 52 52 52 104 104 104 F F F
58022024-02-27T21:46:18.794Z 29 580-599 44 44 44 79 79 79 F F F
58032024-02-27T21:46:18.794ZMax gen: 53, Max flush: 106
58042024-02-27T21:46:18.795Z
58052024-02-27T21:46:18.795ZBegin loop 27
58062024-02-27T21:46:18.795ZDownstairs to restart: 0
58072024-02-27T21:46:18.805Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.202436624Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354}
58082024-02-27T21:46:18.824Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.202716041Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354}
58092024-02-27T21:46:18.824Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.202745498Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354}
58102024-02-27T21:46:18.824Z{"msg":"Crucible 94820df8-5887-4579-afd2-10f7d33bd21c has session id: 91baccf4-45c9-4383-acff-c3673c23d0b0","v":0,"name":"crucible","level":30Crucible runtime is spawned
58112024-02-27T21:46:18.825Z,"time":"2024-02-27T21:46:17.202772604Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58122024-02-27T21:46:18.825Z{"msg":"Upstairs opts: Upstairs UUID: 94820df8-5887-4579-afd2-10f7d33bd21c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.202817599Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58132024-02-27T21:46:18.825Z{"msg":"Crucible stats registered with UUID: 94820df8-5887-4579-afd2-10f7d33bd21c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.202848906Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58142024-02-27T21:46:18.825Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.202864084Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354}
58152024-02-27T21:46:18.825Z{"msg":"94820df8-5887-4579-afd2-10f7d33bd21c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.202929856Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58162024-02-27T21:46:18.825Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.202955573Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58172024-02-27T21:46:18.825Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.202985459Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58182024-02-27T21:46:18.826Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.203022545Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58192024-02-27T21:46:18.826Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.203048042Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58202024-02-27T21:46:18.826Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.203073529Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58212024-02-27T21:46:18.826Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.203120374Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58222024-02-27T21:46:18.826Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.20314843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"io task","client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58232024-02-27T21:46:18.827Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.203177747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"io task","client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58242024-02-27T21:46:18.827Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.203202784Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"io task","client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58252024-02-27T21:46:18.827Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.203343147Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"io task","client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58262024-02-27T21:46:18.827Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.203378383Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"io task","client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58272024-02-27T21:46:18.827Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.203420998Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"io task","client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58282024-02-27T21:46:18.828Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.204139914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58292024-02-27T21:46:18.828Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.204180459Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58302024-02-27T21:46:18.828Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.204196917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58312024-02-27T21:46:18.828Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.204211555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58322024-02-27T21:46:18.829Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.204416501Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58332024-02-27T21:46:18.829Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.204485643Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58342024-02-27T21:46:18.829Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.204906513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58352024-02-27T21:46:18.829Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.204961167Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58362024-02-27T21:46:18.829Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205025659Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58372024-02-27T21:46:18.829Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205383267Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58382024-02-27T21:46:18.830Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205432081Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58392024-02-27T21:46:18.830Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205458998Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58402024-02-27T21:46:18.830Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.20552769Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58412024-02-27T21:46:18.830Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205564995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58422024-02-27T21:46:18.830Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205588293Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58432024-02-27T21:46:18.830Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205601781Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58442024-02-27T21:46:18.831Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205622109Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58452024-02-27T21:46:18.831Z{"msg":"[0]R flush_numbers[0..12]: [104, 101, 92, 65, 101, 99, 106, 72, 94, 102, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205636697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58462024-02-27T21:46:18.831Z{"msg":"[0]R generation[0..12]: [52, 50, 48, 37, 50, 49, 53, 41, 48, 50, 53, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205657315Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58472024-02-27T21:46:18.831Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205671583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58482024-02-27T21:46:18.831Z{"msg":"[1]R flush_numbers[0..12]: [104, 101, 92, 65, 101, 99, 106, 72, 94, 102, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.20569427Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58492024-02-27T21:46:18.832Z{Wait for a query_work_queue command to finish before sending IO
58502024-02-27T21:46:18.832Z"msg":"[1]R generation[0..12]: [52, 50, 48, 37, 50, 49, 53, 41, 48, 50, 53, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205710918Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58512024-02-27T21:46:18.832Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205725287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58522024-02-27T21:46:18.832Z{"msg":"[2]R flush_numbers[0..12]: [104, 101, 92, 65, 101, 99, 106, 72, 94, 102, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205739685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58532024-02-27T21:46:18.832Z{"msg":"[2]R generation[0..12]: [52, 50, 48, 37, 50, 49, 53, 41, 48, 50, 53, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205753863Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58542024-02-27T21:46:18.832Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205775661Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58552024-02-27T21:46:18.832Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205789239Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58562024-02-27T21:46:18.832Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205804437Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58572024-02-27T21:46:18.833Z{"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205825115Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58582024-02-27T21:46:18.833Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205838883Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58592024-02-27T21:46:18.833Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205852252Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58602024-02-27T21:46:18.833Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.20586421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58612024-02-27T21:46:18.833Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205877519Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58622024-02-27T21:46:18.833Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205897267Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58632024-02-27T21:46:18.833Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205910785Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58642024-02-27T21:46:18.833Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205923253Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58652024-02-27T21:46:18.833Z{"msg":"94820df8-5887-4579-afd2-10f7d33bd21c is now active with session: 91baccf4-45c9-4383-acff-c3673c23d0b0","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205943031Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58662024-02-27T21:46:18.834Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.20595672Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354,"session_id":"91baccf4-45c9-4383-acff-c3673c23d0b0"}
58672024-02-27T21:46:18.834Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:17.205969408Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354}
58682024-02-27T21:46:18.834ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58692024-02-27T21:46:18.834ZLoading write count information from file "/tmp/test_repair_verify.data"
58702024-02-27T21:46:18.834ZRun Repair workload
58712024-02-27T21:46:18.834Z01/30 Write block 302 len 24576 data: 6 5 8 7 8 10
58722024-02-27T21:46:18.834Z02/30 Write block 297 len 24576 data: 4 4 3 4 4 7
58732024-02-27T21:46:18.834Z03/30 Read block 470 len 20480
58742024-02-27T21:46:18.834Z04/30 Read block 2 len 8192
58752024-02-27T21:46:18.852Z05/30 Read block 124 len 24576
58762024-02-27T21:46:18.852Z06/30 Read block 347 len 4096
58772024-02-27T21:46:18.852Z07/30 Read block 276 len 4096
58782024-02-27T21:46:18.852Z08/30 Flush
58792024-02-27T21:46:18.852Z09/30 Write block 428 len 40960 data: 3 5 4 4 5 7 9 6 5 5
58802024-02-27T21:46:18.852Z10/30 Read block 18 len 4096
58812024-02-27T21:46:18.853Z11/30 Read block 472 len 4096
58822024-02-27T21:46:18.853Z12/30 Read block 482 len 32768
58832024-02-27T21:46:18.853Z13/30 Write block 451 len 20480 data: 7 6 6 5 4
58842024-02-27T21:46:18.853Z14/30 Read block 210 len 36864
58852024-02-27T21:46:18.853Z15/30 Write block 363 len 24576 data: 5 6 6 6 6 6
58862024-02-27T21:46:18.854Z16/30 Read block 343 len 4096
58872024-02-27T21:46:18.854Z17/30 Flush
58882024-02-27T21:46:18.854Z18/30 Write block 316 len 32768 data: 7 5 5 6 6 4 3 3
58892024-02-27T21:46:18.854Z19/30 Read block 355 len 40960
58902024-02-27T21:46:18.854Z20/30 Write block 397 len 4096 data: 5
58912024-02-27T21:46:18.854Z21/30 Write block 511 len 12288 data: 9 9 8
58922024-02-27T21:46:18.854Z22/30 Read block 452 len 8192
58932024-02-27T21:46:18.855Z23/30 Write block 346 len 28672 data: 6 6 6 6 7 6 6
58942024-02-27T21:46:18.855Z24/30 Flush
58952024-02-27T21:46:18.855Z25/30 Flush
58962024-02-27T21:46:18.855Z26/30 Write block 534 len 16384 data: 5 5 6 5
58972024-02-27T21:46:18.855Z27/30 Write block 65 len 32768 data: 3 3 3 4 3 4 4 5
58982024-02-27T21:46:18.855Z28/30 Write block 362 len 28672 data: 3 6 7 7 7 7 7
58992024-02-27T21:46:18.856Z29/30 Write block 10 len 40960 data: 7 7 8 7 9 8 8 7 5 4
59002024-02-27T21:46:18.856Z30/30 Read block 561 len 36864
59012024-02-27T21:46:18.856Z----------------------------------------------------------------
59022024-02-27T21:46:18.856Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:22
59032024-02-27T21:46:18.857ZStates: Active Active Active
59042024-02-27T21:46:18.857ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59052024-02-27T21:46:18.857Z 9 Acked 1008 Write 10 Done Done Done false
59062024-02-27T21:46:18.857Z 10 Acked 1009 Read 1 Done Done Done false
59072024-02-27T21:46:18.857Z 11 Acked 1010 Read 1 Done Done Done false
59082024-02-27T21:46:18.857Z 12 Acked 1011 Read 8 Done Done Done false
59092024-02-27T21:46:18.858Z 13 Acked 1012 Write 5 Done Done Done false
59102024-02-27T21:46:18.858Z 14 Acked 1013 Read 9 Sent Done Done false
59112024-02-27T21:46:18.858Z 15 Acked 1014 Write 6 Sent Done Done false
59122024-02-27T21:46:18.858Z 16 Acked 1015 Read 1 Sent Done Done false
59132024-02-27T21:46:18.859Z 17 Acked 1016 Flush 0 Sent Done Done false
59142024-02-27T21:46:18.859Z 18 Acked 1017 Write 8 Sent Done Done false
59152024-02-27T21:46:18.859Z 19 Acked 1018 Read 10 Sent Done Done false
59162024-02-27T21:46:18.859Z 20 Acked 1019 Write 1 Sent Done Done false
59172024-02-27T21:46:18.859Z 21 Acked 1020 Write 3 Sent Done Done false
59182024-02-27T21:46:18.860Z 22 Acked 1021 Read 2 Sent Done Done false
59192024-02-27T21:46:18.860Z 23 Acked 1022 Write 7 Sent Done Done false
59202024-02-27T21:46:18.860Z 24 Acked 1023 Flush 0 Sent Done Done false
59212024-02-27T21:46:18.860Z 25 Acked 1024 Flush 0 Sent Done Done false
59222024-02-27T21:46:18.860Z 26 Acked 1025 Write 4 Sent Done Done false
59232024-02-27T21:46:18.860Z 27 Acked 1026 Write 8 Sent Done Done false
59242024-02-27T21:46:18.861Z 28 Acked 1027 Write 7 Sent Done Done false
59252024-02-27T21:46:18.861Z 29 Acked 1028 Write 10 Sent Done Done false
59262024-02-27T21:46:18.861Z 30 Acked 1029 Read 9 Sent Sent Done false
59272024-02-27T21:46:18.861Z STATES DS:0 DS:1 DS:2 TOTAL
59282024-02-27T21:46:18.861Z New 0 0 0 0
59292024-02-27T21:46:18.861Z Sent 17 1 0 18
59302024-02-27T21:46:18.861Z Done 5 21 22 48
59312024-02-27T21:46:18.862Z Skipped 0 0 0 0
59322024-02-27T21:46:18.862Z Error 0 0 0 0
59332024-02-27T21:46:18.862ZLast Flush: 1007 1024 1024
59342024-02-27T21:46:18.862ZDownstairs last five completed: 1007 1006 1005 1004 1003
59352024-02-27T21:46:18.862ZUpstairs last five completed: 30 29 28 27 26
59362024-02-27T21:46:18.863Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:17.23744318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1354}
59372024-02-27T21:46:18.863ZWrote out file "/tmp/test_repair_verify.data"
59382024-02-27T21:46:18.891Z
59392024-02-27T21:46:18.891ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59402024-02-27T21:46:18.891ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59412024-02-27T21:46:18.891Z 0 000-019 52 52 52 104 104 104 F T T <---
59422024-02-27T21:46:18.891Z 1 020-039 50 50 50 101 101 101 F F F
59432024-02-27T21:46:18.891Z 2 040-059 48 48 48 92 92 92 F F F
59442024-02-27T21:46:18.891Z 3 060-079 37 37 37 65 65 65 F T T <---
59452024-02-27T21:46:18.892Z 4 080-099 50 50 50 101 101 101 F F F
59462024-02-27T21:46:18.892Z 5 100-119 49 49 49 99 99 99 F F F
59472024-02-27T21:46:18.892Z 6 120-139 53 53 53 106 106 106 F F F
59482024-02-27T21:46:18.892Z 7 140-159 41 41 41 72 72 72 F F F
59492024-02-27T21:46:18.892Z 8 160-179 48 48 48 94 94 94 F F F
59502024-02-27T21:46:18.892Z 9 180-199 50 50 50 102 102 102 F F F
59512024-02-27T21:46:18.892Z 10 200-219 53 53 53 106 106 106 F F F
59522024-02-27T21:46:18.892Z 11 220-239 50 50 50 101 101 101 F F F
59532024-02-27T21:46:18.892Z 12 240-259 53 53 53 106 106 106 F F F
59542024-02-27T21:46:18.892Z 13 260-279 46 46 46 83 83 83 F F F
59552024-02-27T21:46:18.892Z 14 280-299 54 54 54 107 107 107 F F F
59562024-02-27T21:46:18.892Z 15 300-319 54 54 54 107 109 109 F F F <---
59572024-02-27T21:46:18.893Z 16 320-339 52 54 54 105 109 109 F F F <---
59582024-02-27T21:46:18.893Z 17 340-359 48 54 54 92 109 109 F F F <---
59592024-02-27T21:46:18.893Z 18 360-379 48 54 54 92 108 108 F T T <---
59602024-02-27T21:46:18.893Z 19 380-399 53 54 54 106 109 109 F F F <---
59612024-02-27T21:46:18.893Z 20 400-419 53 53 53 106 106 106 F F F
59622024-02-27T21:46:18.893Z 21 420-439 50 54 54 102 108 108 T F F <---
59632024-02-27T21:46:18.893Z 22 440-459 43 54 54 75 108 108 T F F <---
59642024-02-27T21:46:18.893Z 23 460-479 51 51 51 103 103 103 F F F
59652024-02-27T21:46:18.893Z 24 480-499 50 50 50 101 101 101 F F F
59662024-02-27T21:46:18.893Z 25 500-519 52 54 54 104 109 109 F F F <---
59672024-02-27T21:46:18.893Z 26 520-539 51 51 51 103 103 103 F T T <---
59682024-02-27T21:46:18.893Z 27 540-559 48 48 48 95 95 95 F F F
59692024-02-27T21:46:18.894Z 28 560-579 52 52 52 104 104 104 F F F
59702024-02-27T21:46:18.894Z 29 580-599 44 44 44 79 79 79 F F F
59712024-02-27T21:46:18.894ZMax gen: 54, Max flush: 109
59722024-02-27T21:46:18.894ZError: Difference in extent metadata found!
59732024-02-27T21:46:18.894ZA Difference in extent metadata is expected here
59742024-02-27T21:46:18.895ZOn loop 27
59752024-02-27T21:46:18.895Z
59762024-02-27T21:46:18.895ZVerifying data now
59772024-02-27T21:46:23.431ZNo change after verify
59782024-02-27T21:46:23.431ZLoop: 27 Downstairs dump after verify (and repair):
59792024-02-27T21:46:23.440ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59802024-02-27T21:46:23.440Z 0 000-019 55 55 55 110 110 110 F F F
59812024-02-27T21:46:23.440Z 1 020-039 50 50 50 101 101 101 F F F
59822024-02-27T21:46:23.440Z 2 040-059 48 48 48 92 92 92 F F F
59832024-02-27T21:46:23.440Z 3 060-079 55 55 55 110 110 110 F F F
59842024-02-27T21:46:23.440Z 4 080-099 50 50 50 101 101 101 F F F
59852024-02-27T21:46:23.440Z 5 100-119 49 49 49 99 99 99 F F F
59862024-02-27T21:46:23.440Z 6 120-139 53 53 53 106 106 106 F F F
59872024-02-27T21:46:23.441Z 7 140-159 41 41 41 72 72 72 F F F
59882024-02-27T21:46:23.441Z 8 160-179 48 48 48 94 94 94 F F F
59892024-02-27T21:46:23.441Z 9 180-199 50 50 50 102 102 102 F F F
59902024-02-27T21:46:23.451Z 10 200-219 53 53 53 106 106 106 F F F
59912024-02-27T21:46:23.451Z 11 220-239 50 50 50 101 101 101 F F F
59922024-02-27T21:46:23.451Z 12 240-259 53 53 53 106 106 106 F F F
59932024-02-27T21:46:23.451Z 13 260-279 46 46 46 83 83 83 F F F
59942024-02-27T21:46:23.451Z 14 280-299 54 54 54 107 107 107 F F F
59952024-02-27T21:46:23.451Z 15 300-319 54 54 54 109 109 109 F F F
59962024-02-27T21:46:23.451Z 16 320-339 54 54 54 109 109 109 F F F
59972024-02-27T21:46:23.451Z 17 340-359 54 54 54 109 109 109 F F F
59982024-02-27T21:46:23.452Z 18 360-379 55 55 55 110 110 110 F F F
59992024-02-27T21:46:23.452Z 19 380-399 54 54 54 109 109 109 F F F
60002024-02-27T21:46:23.452Z 20 400-419 53 53 53 106 106 106 F F F
60012024-02-27T21:46:23.452Z 21 420-439 54 54 54 108 108 108 F F F
60022024-02-27T21:46:23.452Z 22 440-459 54 54 54 108 108 108 F F F
60032024-02-27T21:46:23.452Z 23 460-479 51 51 51 103 103 103 F F F
60042024-02-27T21:46:23.453Z 24 480-499 50 50 50 101 101 101 F F F
60052024-02-27T21:46:23.453Z 25 500-519 54 54 54 109 109 109 F F F
60062024-02-27T21:46:23.453Z 26 520-539 55 55 55 110 110 110 F F F
60072024-02-27T21:46:23.453Z 27 540-559 48 48 48 95 95 95 F F F
60082024-02-27T21:46:23.453Z 28 560-579 52 52 52 104 104 104 F F F
60092024-02-27T21:46:23.453Z 29 580-599 44 44 44 79 79 79 F F F
60102024-02-27T21:46:23.453ZMax gen: 55, Max flush: 110
60112024-02-27T21:46:23.453Z
60122024-02-27T21:46:23.453ZBegin loop 28
60132024-02-27T21:46:23.453ZDownstairs to restart: 0
60142024-02-27T21:46:23.887Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.860552163Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362}
60152024-02-27T21:46:23.887Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.860856257Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362}
60162024-02-27T21:46:23.887ZCrucible runtime is spawned
60172024-02-27T21:46:23.887Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.860907182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362}
60182024-02-27T21:46:23.887Z{"msg":"Crucible 9c621254-26f7-4eae-b5ac-6d09b7a173a4 has session id: 866ac6a5-d58c-4aef-8aae-6e25aa99a28c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.860949226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60192024-02-27T21:46:23.887Z{"msg":"Upstairs opts: Upstairs UUID: 9c621254-26f7-4eae-b5ac-6d09b7a173a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861015619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60202024-02-27T21:46:23.887Z{"msg":"Crucible stats registered with UUID: 9c621254-26f7-4eae-b5ac-6d09b7a173a4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861045155Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60212024-02-27T21:46:23.887Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861083991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362}
60222024-02-27T21:46:23.887Z{"msg":"9c621254-26f7-4eae-b5ac-6d09b7a173a4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861111627Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60232024-02-27T21:46:23.887Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861152513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60242024-02-27T21:46:23.887Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861190338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60252024-02-27T21:46:23.887Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861222994Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60262024-02-27T21:46:23.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861304205Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60272024-02-27T21:46:23.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861354519Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60282024-02-27T21:46:23.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861383075Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60292024-02-27T21:46:23.888Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861420891Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"io task","client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60302024-02-27T21:46:23.888Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861464886Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"io task","client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60312024-02-27T21:46:23.888Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861497592Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"io task","client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60322024-02-27T21:46:23.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861534098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"io task","client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60332024-02-27T21:46:23.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861561645Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"io task","client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60342024-02-27T21:46:23.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.861614378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"io task","client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60352024-02-27T21:46:23.888Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.862100251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60362024-02-27T21:46:23.888Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.862139956Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60372024-02-27T21:46:23.888Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.862156384Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60382024-02-27T21:46:23.888Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.862170063Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60392024-02-27T21:46:23.888Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.862183761Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60402024-02-27T21:46:23.888Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.862203589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60412024-02-27T21:46:23.888Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.862667204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60422024-02-27T21:46:23.888Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.862801228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60432024-02-27T21:46:23.888Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.862838374Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60442024-02-27T21:46:23.888Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863148487Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60452024-02-27T21:46:23.888Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863197982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60462024-02-27T21:46:23.888Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863230578Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60472024-02-27T21:46:23.888Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863262264Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60482024-02-27T21:46:23.888Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863284491Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60492024-02-27T21:46:23.888Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863324767Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60502024-02-27T21:46:23.888Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863348834Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60512024-02-27T21:46:23.889Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863376471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60522024-02-27T21:46:23.889Z{"msg":"[0]R flush_numbers[0..12]: [110, 101, 92, 110, 101, 99, 106, 72, 94, 102, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863402268Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60532024-02-27T21:46:23.889Z{"msg":"[0]R generation[0..12]: [55, 50, 48, 55, 50, 49, 53, 41, 48, 50, 53, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863435834Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60542024-02-27T21:46:23.889Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863463001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60552024-02-27T21:46:23.889Z{"msg":"[1]R flush_numbers[0..12]: [110, 101, 92, 110, 101, 99, 106, 72, 94, 102, 106, 101]","v":Wait for a query_work_queue command to finish before sending IO
60562024-02-27T21:46:23.889Z0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863508425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60572024-02-27T21:46:23.889Z{"msg":"[1]R generation[0..12]: [55, 50, 48, 55, 50, 49, 53, 41, 48, 50, 53, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863546281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60582024-02-27T21:46:23.889Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863569898Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60592024-02-27T21:46:23.889Z{"msg":"[2]R flush_numbers[0..12]: [110, 101, 92, 110, 101, 99, 106, 72, 94, 102, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863605194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60602024-02-27T21:46:23.889Z{"msg":"[2]R generation[0..12]: [55, 50, 48, 55, 50, 49, 53, 41, 48, 50, 53, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863644389Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60612024-02-27T21:46:23.889Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863679615Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60622024-02-27T21:46:23.889Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863696753Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60632024-02-27T21:46:23.889Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.86372405Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60642024-02-27T21:46:23.889Z{"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863760725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60652024-02-27T21:46:23.889Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863791692Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60662024-02-27T21:46:23.889Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.86380742Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60672024-02-27T21:46:23.889Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863839166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60682024-02-27T21:46:23.889Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863874922Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60692024-02-27T21:46:23.889Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.8638923Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60702024-02-27T21:46:23.889Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863919027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60712024-02-27T21:46:23.889Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863954023Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60722024-02-27T21:46:23.889Z{"msg":"9c621254-26f7-4eae-b5ac-6d09b7a173a4 is now active with session: 866ac6a5-d58c-4aef-8aae-6e25aa99a28c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.863981729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60732024-02-27T21:46:23.889Z{"msg":"Set Active after no reconciliation","v"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
60742024-02-27T21:46:23.890Z:0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.864022105Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362,"session_id":"866ac6a5-d58c-4aef-8aae-6e25aa99a28c"}
60752024-02-27T21:46:23.890Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:21.86405724Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362}
60762024-02-27T21:46:23.890ZLoading write count information from file "/tmp/test_repair_verify.data"
60772024-02-27T21:46:23.890ZRun Repair workload
60782024-02-27T21:46:23.890Z01/30 Write block 511 len 8192 data: 10 10
60792024-02-27T21:46:23.890Z02/30 Write block 133 len 20480 data: 2 2 4 4 4
60802024-02-27T21:46:23.890Z03/30 Read block 2 len 12288
60812024-02-27T21:46:23.890Z04/30 Write block 56 len 12288 data: 8 8 7
60822024-02-27T21:46:23.890Z05/30 Read block 107 len 28672
60832024-02-27T21:46:23.890Z06/30 Write block 442 len 20480 data: 2 4 4 4 5
60842024-02-27T21:46:23.890Z07/30 Read block 488 len 8192
60852024-02-27T21:46:23.890Z08/30 Read block 38 len 20480
60862024-02-27T21:46:23.890Z09/30 Read block 473 len 4096
60872024-02-27T21:46:23.890Z10/30 Read block 455 len 40960
60882024-02-27T21:46:23.890Z11/30 Read block 587 len 32768
60892024-02-27T21:46:23.890Z12/30 Read block 444 len 40960
60902024-02-27T21:46:23.890Z13/30 Write block 513 len 32768 data: 9 7 6 7 7 5 5 5
60912024-02-27T21:46:23.890Z14/30 Read block 149 len 32768
60922024-02-27T21:46:23.890Z15/30 Read block 375 len 8192
60932024-02-27T21:46:23.890Z16/30 Write block 216 len 20480 data: 5 8 9 8 7
60942024-02-27T21:46:23.890Z17/30 Read block 73 len 40960
60952024-02-27T21:46:23.890Z18/30 Read block 379 len 36864
60962024-02-27T21:46:23.890Z19/30 Read block 181 len 8192
60972024-02-27T21:46:23.890Z20/30 Write block 75 len 36864 data: 5 4 5 5 5 3 3 3 3
60982024-02-27T21:46:23.890Z21/30 Write block 292 len 16384 data: 6 5 5 5
60992024-02-27T21:46:23.891Z22/30 Read block 477 len 40960
61002024-02-27T21:46:23.891Z23/30 Write block 264 len 36864 data: 3 2 2 4 4 4 4 5 5
61012024-02-27T21:46:23.891Z24/30 Read block 533 len 32768
61022024-02-27T21:46:23.891Z25/30 Read block 283 len 4096
61032024-02-27T21:46:23.891Z26/30 Read block 300 len 36864
61042024-02-27T21:46:23.891Z27/30 Read block 188 len 8192
61052024-02-27T21:46:23.891Z28/30 Read block 461 len 20480
61062024-02-27T21:46:23.891Z29/30 Read block 177 len 20480
61072024-02-27T21:46:23.891Z30/30 Write block 241 len 24576 data: 3 3 4 5 5 5
61082024-02-27T21:46:23.891Z----------------------------------------------------------------
61092024-02-27T21:46:23.891Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61102024-02-27T21:46:23.891ZStates: Active Active Active
61112024-02-27T21:46:23.891ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61122024-02-27T21:46:23.891Z 1 Acked 1000 Write 2 Done Done Done false
61132024-02-27T21:46:23.891Z 2 Acked 1001 Write 5 Done Done Done false
61142024-02-27T21:46:23.896Z 3 Acked 1002 Read 3 Done Done Done false
61152024-02-27T21:46:23.896Z 4 Acked 1003 Write 3 Done Done Done false
61162024-02-27T21:46:23.896Z 5 Acked 1004 Read 7 Done Done Done false
61172024-02-27T21:46:23.896Z 6 Acked 1005 Write 5 Sent Done Done false
61182024-02-27T21:46:23.896Z 7 Acked 1006 Read 2 Sent Done Done false
61192024-02-27T21:46:23.896Z 8 Acked 1007 Read 5 Sent Done Done false
61202024-02-27T21:46:23.896Z 9 Acked 1008 Read 1 Sent Done Done false
61212024-02-27T21:46:23.896Z 10 Acked 1009 Read 10 Sent Done Done false
61222024-02-27T21:46:23.897Z 11 Acked 1010 Read 8 Sent Done Done false
61232024-02-27T21:46:23.897Z 12 Acked 1011 Read 10 Sent Done Done false
61242024-02-27T21:46:23.897Z 13 Acked 1012 Write 8 Sent Done Done false
61252024-02-27T21:46:23.897Z 14 Acked 1013 Read 8 Sent Done Done false
61262024-02-27T21:46:23.897Z 15 Acked 1014 Read 2 Sent Done Done false
61272024-02-27T21:46:23.897Z 16 Acked 1015 Write 5 Sent Done Done false
61282024-02-27T21:46:23.897Z 17 Acked 1016 Read 10 Sent Done Done false
61292024-02-27T21:46:23.897Z 18 Acked 1017 Read 9 Sent Done Done false
61302024-02-27T21:46:23.897Z 19 Acked 1018 Read 2 Sent Done Done false
61312024-02-27T21:46:23.897Z 20 Acked 1019 Write 9 Sent Done Done false
61322024-02-27T21:46:23.897Z 21 Acked 1020 Write 4 Sent Done Done false
61332024-02-27T21:46:23.897Z 22 Acked 1021 Read 10 Sent Done Done false
61342024-02-27T21:46:23.897Z 23 Acked 1022 Write 9 Sent Done Done false
61352024-02-27T21:46:23.897Z 24 Acked 1023 Read 8 Sent Done Done false
61362024-02-27T21:46:23.897Z 25 Acked 1024 Read 1 Sent Done Done false
61372024-02-27T21:46:23.897Z 26 Acked 1025 Read 9 Sent Done Done false
61382024-02-27T21:46:23.897Z 27 Acked 1026 Read 2 Sent Done Done false
61392024-02-27T21:46:23.897Z 28 Acked 1027 Read 5 Sent Done Done false
61402024-02-27T21:46:23.897Z 29 Acked 1028 Read 5 Sent Done Done false
61412024-02-27T21:46:23.897Z 30 Acked 1029 Write 6 Sent Sent Sent false
61422024-02-27T21:46:23.897Z STATES DS:0 DS:1 DS:2 TOTAL
61432024-02-27T21:46:23.897Z New 0 0 0 0
61442024-02-27T21:46:23.897Z Sent 25 1 1 27
61452024-02-27T21:46:23.897Z Done 5 29 29 63
61462024-02-27T21:46:23.897Z Skipped 0 0 0 0
61472024-02-27T21:46:23.898Z Error 0 0 0 0
61482024-02-27T21:46:23.898ZLast Flush: 0 0 0
61492024-02-27T21:46:23.898ZDownstairs last five completed:
61502024-02-27T21:46:23.898ZUpstairs last five completed: 30 29 28 27 26
61512024-02-27T21:46:23.898Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:21.891424387Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1362}
61522024-02-27T21:46:23.898ZWrote out file "/tmp/test_repair_verify.data"
61532024-02-27T21:46:23.898Z
61542024-02-27T21:46:23.898ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61552024-02-27T21:46:23.898ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61562024-02-27T21:46:23.898Z 0 000-019 55 55 55 110 110 110 F F F
61572024-02-27T21:46:23.898Z 1 020-039 50 50 50 101 101 101 F F F
61582024-02-27T21:46:23.898Z 2 040-059 48 48 48 92 92 92 T T T
61592024-02-27T21:46:23.898Z 3 060-079 55 55 55 110 110 110 F T T <---
61602024-02-27T21:46:23.898Z 4 080-099 50 50 50 101 101 101 F T T <---
61612024-02-27T21:46:23.898Z 5 100-119 49 49 49 99 99 99 F F F
61622024-02-27T21:46:23.898Z 6 120-139 53 53 53 106 106 106 T T T
61632024-02-27T21:46:23.898Z 7 140-159 41 41 41 72 72 72 F F F
61642024-02-27T21:46:23.898Z 8 160-179 48 48 48 94 94 94 F F F
61652024-02-27T21:46:23.898Z 9 180-199 50 50 50 102 102 102 F F F
61662024-02-27T21:46:23.898Z 10 200-219 53 53 53 106 106 106 F T T <---
61672024-02-27T21:46:23.898Z 11 220-239 50 50 50 101 101 101 F T T <---
61682024-02-27T21:46:23.898Z 12 240-259 53 53 53 106 106 106 F T T <---
61692024-02-27T21:46:23.898Z 13 260-279 46 46 46 83 83 83 F T T <---
61702024-02-27T21:46:23.898Z 14 280-299 54 54 54 107 107 107 F T T <---
61712024-02-27T21:46:23.898Z 15 300-319 54 54 54 109 109 109 F F F
61722024-02-27T21:46:23.899Z 16 320-339 54 54 54 109 109 109 F F F
61732024-02-27T21:46:23.899Z 17 340-359 54 54 54 109 109 109 F F F
61742024-02-27T21:46:23.899Z 18 360-379 55 55 55 110 110 110 F F F
61752024-02-27T21:46:23.899Z 19 380-399 54 54 54 109 109 109 F F F
61762024-02-27T21:46:23.899Z 20 400-419 53 53 53 106 106 106 F F F
61772024-02-27T21:46:23.899Z 21 420-439 54 54 54 108 108 108 F F F
61782024-02-27T21:46:23.899Z 22 440-459 54 54 54 108 108 108 F T T <---
61792024-02-27T21:46:23.899Z 23 460-479 51 51 51 103 103 103 F F F
61802024-02-27T21:46:23.899Z 24 480-499 50 50 50 101 101 101 F F F
61812024-02-27T21:46:23.899Z 25 500-519 54 54 54 109 109 109 T T T
61822024-02-27T21:46:23.899Z 26 520-539 55 55 55 110 110 110 F T T <---
61832024-02-27T21:46:23.899Z 27 540-559 48 48 48 95 95 95 F F F
61842024-02-27T21:46:23.899Z 28 560-579 52 52 52 104 104 104 F F F
61852024-02-27T21:46:23.899Z 29 580-599 44 44 44 79 79 79 F F F
61862024-02-27T21:46:23.899ZMax gen: 55, Max flush: 110
61872024-02-27T21:46:23.899ZError: Difference in extent metadata found!
61882024-02-27T21:46:23.899ZA Difference in extent metadata is expected here
61892024-02-27T21:46:23.899ZOn loop 28
61902024-02-27T21:46:23.899Z
61912024-02-27T21:46:23.899ZVerifying data now
61922024-02-27T21:46:28.198ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
61932024-02-27T21:46:28.217Z516c516,517
61942024-02-27T21:46:28.217Z< 8,
61952024-02-27T21:46:28.217Z---
61962024-02-27T21:46:28.218Z> 9,
61972024-02-27T21:46:28.218Z> 7,
61982024-02-27T21:46:28.218Z517a519,520
61992024-02-27T21:46:28.218Z> 7,
62002024-02-27T21:46:28.218Z> 7,
62012024-02-27T21:46:28.218Z519,522d521
62022024-02-27T21:46:28.218Z< 6,
62032024-02-27T21:46:28.218Z< 6,
62042024-02-27T21:46:28.218Z< 4,
62052024-02-27T21:46:28.218Z< 4,
62062024-02-27T21:46:28.218Z523a523
62072024-02-27T21:46:28.218Z> 5,
62082024-02-27T21:46:28.218Zdiff found after verify
62092024-02-27T21:46:28.218ZLoop: 28 Downstairs dump after verify (and repair):
62102024-02-27T21:46:28.640ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62112024-02-27T21:46:28.640Z 0 000-019 55 55 55 110 110 110 F F F
62122024-02-27T21:46:28.640Z 1 020-039 50 50 50 101 101 101 F F F
62132024-02-27T21:46:28.640Z 2 040-059 56 56 56 111 111 111 F F F
62142024-02-27T21:46:28.640Z 3 060-079 56 56 56 111 111 111 F F F
62152024-02-27T21:46:28.641Z 4 080-099 56 56 56 111 111 111 F F F
62162024-02-27T21:46:28.641Z 5 100-119 49 49 49 99 99 99 F F F
62172024-02-27T21:46:28.641Z 6 120-139 56 56 56 111 111 111 F F F
62182024-02-27T21:46:28.641Z 7 140-159 41 41 41 72 72 72 F F F
62192024-02-27T21:46:28.641Z 8 160-179 48 48 48 94 94 94 F F F
62202024-02-27T21:46:28.641Z 9 180-199 50 50 50 102 102 102 F F F
62212024-02-27T21:46:28.641Z 10 200-219 56 56 56 111 111 111 F F F
62222024-02-27T21:46:28.641Z 11 220-239 56 56 56 111 111 111 F F F
62232024-02-27T21:46:28.642Z 12 240-259 56 56 56 111 111 111 F F F
62242024-02-27T21:46:28.642Z 13 260-279 56 56 56 111 111 111 F F F
62252024-02-27T21:46:28.642Z 14 280-299 56 56 56 111 111 111 F F F
62262024-02-27T21:46:28.642Z 15 300-319 54 54 54 109 109 109 F F F
62272024-02-27T21:46:28.642Z 16 320-339 54 54 54 109 109 109 F F F
62282024-02-27T21:46:28.642Z 17 340-359 54 54 54 109 109 109 F F F
62292024-02-27T21:46:28.642Z 18 360-379 55 55 55 110 110 110 F F F
62302024-02-27T21:46:28.642Z 19 380-399 54 54 54 109 109 109 F F F
62312024-02-27T21:46:28.642Z 20 400-419 53 53 53 106 106 106 F F F
62322024-02-27T21:46:28.642Z 21 420-439 54 54 54 108 108 108 F F F
62332024-02-27T21:46:28.642Z 22 440-459 56 56 56 111 111 111 F F F
62342024-02-27T21:46:28.642Z 23 460-479 51 51 51 103 103 103 F F F
62352024-02-27T21:46:28.642Z 24 480-499 50 50 50 101 101 101 F F F
62362024-02-27T21:46:28.643Z 25 500-519 56 56 56 111 111 111 F F F
62372024-02-27T21:46:28.643Z 26 520-539 56 56 56 111 111 111 F F F
62382024-02-27T21:46:28.643Z 27 540-559 48 48 48 95 95 95 F F F
62392024-02-27T21:46:28.643Z 28 560-579 52 52 52 104 104 104 F F F
62402024-02-27T21:46:28.643Z 29 580-599 44 44 44 79 79 79 F F F
62412024-02-27T21:46:28.643ZMax gen: 56, Max flush: 111
62422024-02-27T21:46:28.643Z
62432024-02-27T21:46:28.643ZBegin loop 29
62442024-02-27T21:46:28.643ZDownstairs to restart: 0
62452024-02-27T21:46:28.669Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.635607829Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373}
62462024-02-27T21:46:28.669Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.635870948Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373}
62472024-02-27T21:46:28.669Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.635891675Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373}
62482024-02-27T21:46:28.669Z{"msg":"Crucible 3b360b7d-56dd-44ce-a1d7-146efa281f26 has session id: 269529f7-72cd-40d3-97a0-b7f9e91ddc98","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.63593554Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62492024-02-27T21:46:28.670ZCrucible runtime is spawned
62502024-02-27T21:46:28.670Z{"msg":"Upstairs opts: Upstairs UUID: 3b360b7d-56dd-44ce-a1d7-146efa281f26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.635993273Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62512024-02-27T21:46:28.670Z{"msg":"Crucible stats registered with UUID: 3b360b7d-56dd-44ce-a1d7-146efa281f26","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636031989Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62522024-02-27T21:46:28.670Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636056656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373}
62532024-02-27T21:46:28.670Z{"msg":"3b360b7d-56dd-44ce-a1d7-146efa281f26 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636173212Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62542024-02-27T21:46:28.670Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636220167Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62552024-02-27T21:46:28.670Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636248693Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62562024-02-27T21:46:28.670Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636271581Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62572024-02-27T21:46:28.670Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636313376Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62582024-02-27T21:46:28.670Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636344722Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62592024-02-27T21:46:28.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636384137Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62602024-02-27T21:46:28.671Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636409454Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62612024-02-27T21:46:28.671Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636440281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62622024-02-27T21:46:28.671Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636489145Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62632024-02-27T21:46:28.671Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636525341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62642024-02-27T21:46:28.671Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.636560867Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62652024-02-27T21:46:28.672Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.63661469Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62662024-02-27T21:46:28.672Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.637264514Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62672024-02-27T21:46:28.672Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.637318597Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62682024-02-27T21:46:28.672Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.637348744Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62692024-02-27T21:46:28.672Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.637397638Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62702024-02-27T21:46:28.672Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.637439003Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62712024-02-27T21:46:28.672Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.637482278Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62722024-02-27T21:46:28.672Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.637866433Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62732024-02-27T21:46:28.672Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.637918377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62742024-02-27T21:46:28.672Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638039012Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62752024-02-27T21:46:28.672Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638459233Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62762024-02-27T21:46:28.673Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638517406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62772024-02-27T21:46:28.673Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638554622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62782024-02-27T21:46:28.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.63857095Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62792024-02-27T21:46:28.673Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638593037Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62802024-02-27T21:46:28.673Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638607845Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62812024-02-27T21:46:28.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638627403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62822024-02-27T21:46:28.673Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638641322Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62832024-02-27T21:46:28.673Z{"msg":"[0]R flush_numbers[0..12]: [110, 101, 111, 111, 111, 99, 111, 72, 94, 102, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.63865603Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62842024-02-27T21:46:28.674ZWait for a query_work_queue command to finish before sending IO
62852024-02-27T21:46:28.674Z{"msg":"[0]R generation[0..12]: [55, 50, 56, 56, 56, 49, 56, 41, 48, 50, 56, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638679717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62862024-02-27T21:46:28.674Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638694615Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62872024-02-27T21:46:28.674Z{"msg":"[1]R flush_numbers[0..12]: [110, 101, 111, 111, 111, 99, 111, 72, 94, 102, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638721402Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62882024-02-27T21:46:28.674Z{"msg":"[1]R generation[0..12]: [55, 50, 56, 56, 56, 49, 56, 41, 48, 50, 56, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.63873633Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62892024-02-27T21:46:28.674Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638764147Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62902024-02-27T21:46:28.674Z{"msg":"[2]R flush_numbers[0..12]: [110, 101, 111, 111, 111, 99, 111, 72, 94, 102, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638791414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62912024-02-27T21:46:28.674Z{"msg":"[2]R generation[0..12]: [55, 50, 56, 56, 56, 49, 56, 41, 48, 50, 56, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.63882201Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62922024-02-27T21:46:28.674Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638857776Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62932024-02-27T21:46:28.674Z{"msg":"Max found gen is 57","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638897811Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62942024-02-27T21:46:28.675Z{"msg":"Generation requested: 58 >= found:57","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638926158Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62952024-02-27T21:46:28.675Z{"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.638956284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62962024-02-27T21:46:28.675Z{"msg":"All extents match","v":0,"name":"crucible","level":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
62972024-02-27T21:46:28.675Z30,"time":"2024-02-27T21:46:26.638984621Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"":"downstairs","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62982024-02-27T21:46:28.675Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.639038285Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
62992024-02-27T21:46:28.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.639062932Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"0","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
63002024-02-27T21:46:28.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.639089859Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"1","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
63012024-02-27T21:46:28.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.639113636Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"client":"2","":"downstairs","session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
63022024-02-27T21:46:28.675Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.639141383Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
63032024-02-27T21:46:28.676Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.63916479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
63042024-02-27T21:46:28.676Z{"msg":"3b360b7d-56dd-44ce-a1d7-146efa281f26 is now active with session: 269529f7-72cd-40d3-97a0-b7f9e91ddc98","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.639192217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
63052024-02-27T21:46:28.676Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.639213984Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373,"session_id":"269529f7-72cd-40d3-97a0-b7f9e91ddc98"}
63062024-02-27T21:46:28.676Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:26.639240171Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373}
63072024-02-27T21:46:28.676ZLoading write count information from file "/tmp/test_repair_verify.data"
63082024-02-27T21:46:28.676ZRun Repair workload
63092024-02-27T21:46:28.677Z01/30 Write block 412 len 24576 data: 4 4 4 4 5 7
63102024-02-27T21:46:28.677Z02/30 Write block 551 len 16384 data: 6 4 5 7
63112024-02-27T21:46:28.677Z03/30 Flush
63122024-02-27T21:46:28.677Z04/30 Write block 65 len 20480 data: 4 4 4 5 4
63132024-02-27T21:46:28.677Z05/30 Write block 568 len 28672 data: 3 4 4 5 5 7 5
63142024-02-27T21:46:28.677Z06/30 Write block 317 len 32768 data: 6 6 7 7 5 4 4 3
63152024-02-27T21:46:28.678Z07/30 Write block 156 len 28672 data: 7 7 7 7 6 6 5
63162024-02-27T21:46:28.678Z08/30 Read block 203 len 4096
63172024-02-27T21:46:28.678Z09/30 Read block 213 len 8192
63182024-02-27T21:46:28.678Z10/30 Write block 363 len 36864 data: 7 8 8 8 8 8 5 6 6
63192024-02-27T21:46:28.679Z11/30 Write block 178 len 24576 data: 7 5 6 7 7 7
63202024-02-27T21:46:28.679Z12/30 Write block 468 len 8192 data: 5 4
63212024-02-27T21:46:28.679Z13/30 Write block 229 len 32768 data: 9 9 10 11 9 8 8 8
63222024-02-27T21:46:28.679Z14/30 Read block 308 len 36864
63232024-02-27T21:46:28.679Z15/30 Write block 188 len 12288 data: 7 7 7
63242024-02-27T21:46:28.679Z16/30 Flush
63252024-02-27T21:46:28.680Z17/30 Write block 546 len 16384 data: 2 2 5 5
63262024-02-27T21:46:28.680Z18/30 Write block 208 len 28672 data: 6 6 6 6 6 4 4
63272024-02-27T21:46:28.680Z19/30 Read block 78 len 24576
63282024-02-27T21:46:28.680Z20/30 Read block 326 len 16384
63292024-02-27T21:46:28.680Z21/30 Write block 424 len 4096 data: 3
63302024-02-27T21:46:28.680Z22/30 Read block 348 len 32768
63312024-02-27T21:46:28.680Z23/30 Read block 441 len 8192
63322024-02-27T21:46:28.680Z24/30 Read block 227 len 28672
63332024-02-27T21:46:28.681Z25/30 Read block 318 len 36864
63342024-02-27T21:46:28.681Z26/30 Read block 199 len 32768
63352024-02-27T21:46:28.681Z27/30 Read block 571 len 20480
63362024-02-27T21:46:28.681Z28/30 Read block 108 len 32768
63372024-02-27T21:46:28.681Z29/30 Read block 112 len 36864
63382024-02-27T21:46:28.681Z30/30 Read block 555 len 32768
63392024-02-27T21:46:28.681Z----------------------------------------------------------------
63402024-02-27T21:46:28.681Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:27
63412024-02-27T21:46:28.681ZStates: Active Active Active
63422024-02-27T21:46:28.681ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63432024-02-27T21:46:28.681Z 4 Acked 1003 Write 5 Done Done Done false
63442024-02-27T21:46:28.681Z 5 Acked 1004 Write 7 Done Done Done false
63452024-02-27T21:46:30.041Z 6 Acked 1005 Write 8 Sent Done Done false
63462024-02-27T21:46:30.041Z 7 Acked 1006 Write 7 Sent Done Done false
63472024-02-27T21:46:30.041Z 8 Acked 1007 Read 1 Sent Done Done false
63482024-02-27T21:46:30.041Z 9 Acked 1008 Read 2 Sent Done Done false
63492024-02-27T21:46:30.041Z 10 Acked 1009 Write 9 Sent Done Done false
63502024-02-27T21:46:30.041Z 11 Acked 1010 Write 6 Sent Done Done false
63512024-02-27T21:46:30.041Z 12 Acked 1011 Write 2 Sent Done Done false
63522024-02-27T21:46:30.041Z 13 Acked 1012 Write 8 Sent Done Done false
63532024-02-27T21:46:30.042Z 14 Acked 1013 Read 9 Sent Done Done false
63542024-02-27T21:46:30.042Z 15 Acked 1014 Write 3 Sent Done Done false
63552024-02-27T21:46:30.042Z 16 Acked 1015 Flush 0 Sent Done Done false
63562024-02-27T21:46:30.042Z 17 Acked 1016 Write 4 Sent Done Done false
63572024-02-27T21:46:30.042Z 18 Acked 1017 Write 7 Sent Done Done false
63582024-02-27T21:46:30.042Z 19 Acked 1018 Read 6 Sent Done Done false
63592024-02-27T21:46:30.042Z 20 Acked 1019 Read 4 Sent Done Done false
63602024-02-27T21:46:30.042Z 21 Acked 1020 Write 1 Sent Done Done false
63612024-02-27T21:46:30.042Z 22 Acked 1021 Read 8 Sent Done Done false
63622024-02-27T21:46:30.042Z 23 Acked 1022 Read 2 Sent Done Done false
63632024-02-27T21:46:30.042Z 24 Acked 1023 Read 7 Sent Done Done false
63642024-02-27T21:46:30.042Z 25 Acked 1024 Read 9 Sent Done Done false
63652024-02-27T21:46:30.043Z 26 Acked 1025 Read 8 Sent Done Done false
63662024-02-27T21:46:30.043Z 27 Acked 1026 Read 5 Sent Done Done false
63672024-02-27T21:46:30.043Z 28 Acked 1027 Read 8 Sent Done Done false
63682024-02-27T21:46:30.043Z 29 Acked 1028 Read 9 Sent Done Done false
63692024-02-27T21:46:30.043Z 30 Acked 1029 Read 8 Sent Sent Done false
63702024-02-27T21:46:30.043Z STATES DS:0 DS:1 DS:2 TOTAL
63712024-02-27T21:46:30.043Z New 0 0 0 0
63722024-02-27T21:46:30.043Z Sent 25 1 0 26
63732024-02-27T21:46:30.044Z Done 2 26 27 55
63742024-02-27T21:46:30.044Z Skipped 0 0 0 0
63752024-02-27T21:46:30.044Z Error 0 0 0 0
63762024-02-27T21:46:30.044ZLast Flush: 1002 1015 1015
63772024-02-27T21:46:30.044ZDownstairs last five completed: 1002 1001 1000
63782024-02-27T21:46:30.044ZUpstairs last five completed: 30 29 28 27 26
63792024-02-27T21:46:30.044Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:26.668586854Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1373}
63802024-02-27T21:46:30.045ZWrote out file "/tmp/test_repair_verify.data"
63812024-02-27T21:46:30.045Z
63822024-02-27T21:46:30.045ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63832024-02-27T21:46:30.045ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63842024-02-27T21:46:30.045Z 0 000-019 55 55 55 110 110 110 F F F
63852024-02-27T21:46:30.045Z 1 020-039 50 50 50 101 101 101 F F F
63862024-02-27T21:46:30.045Z 2 040-059 56 56 56 111 111 111 F F F
63872024-02-27T21:46:30.045Z 3 060-079 56 58 58 111 113 113 T F F <---
63882024-02-27T21:46:30.046Z 4 080-099 56 56 56 111 111 111 F F F
63892024-02-27T21:46:30.046Z 5 100-119 49 49 49 99 99 99 F F F
63902024-02-27T21:46:30.046Z 6 120-139 56 56 56 111 111 111 F F F
63912024-02-27T21:46:30.046Z 7 140-159 41 58 58 72 113 113 F F F <---
63922024-02-27T21:46:30.046Z 8 160-179 48 58 58 94 113 113 F F F <---
63932024-02-27T21:46:30.046Z 9 180-199 50 58 58 102 113 113 F F F <---
63942024-02-27T21:46:30.047Z 10 200-219 56 56 56 111 111 111 F T T <---
63952024-02-27T21:46:30.047Z 11 220-239 56 58 58 111 113 113 F F F <---
63962024-02-27T21:46:30.047Z 12 240-259 56 56 56 111 111 111 F F F
63972024-02-27T21:46:30.047Z 13 260-279 56 56 56 111 111 111 F F F
63982024-02-27T21:46:30.047Z 14 280-299 56 56 56 111 111 111 F F F
63992024-02-27T21:46:30.047Z 15 300-319 54 58 58 109 113 113 F F F <---
64002024-02-27T21:46:30.047Z 16 320-339 54 58 58 109 113 113 F F F <---
64012024-02-27T21:46:30.047Z 17 340-359 54 54 54 109 109 109 F F F
64022024-02-27T21:46:30.048Z 18 360-379 55 58 58 110 113 113 F F F <---
64032024-02-27T21:46:30.048Z 19 380-399 54 54 54 109 109 109 F F F
64042024-02-27T21:46:30.048Z 20 400-419 58 58 58 112 112 112 F F F
64052024-02-27T21:46:30.048Z 21 420-439 54 54 54 108 108 108 F T T <---
64062024-02-27T21:46:30.048Z 22 440-459 56 56 56 111 111 111 F F F
64072024-02-27T21:46:30.048Z 23 460-479 51 58 58 103 113 113 F F F <---
64082024-02-27T21:46:30.048Z 24 480-499 50 50 50 101 101 101 F F F
64092024-02-27T21:46:30.049Z 25 500-519 56 56 56 111 111 111 F F F
64102024-02-27T21:46:30.049Z 26 520-539 56 56 56 111 111 111 F F F
64112024-02-27T21:46:30.049Z 27 540-559 58 58 58 112 112 112 F T T <---
64122024-02-27T21:46:30.049Z 28 560-579 52 58 58 104 113 113 T F F <---
64132024-02-27T21:46:30.049Z 29 580-599 44 44 44 79 79 79 F F F
64142024-02-27T21:46:30.049ZMax gen: 58, Max flush: 113
64152024-02-27T21:46:30.049ZError: Difference in extent metadata found!
64162024-02-27T21:46:30.049ZA Difference in extent metadata is expected here
64172024-02-27T21:46:30.050ZOn loop 29
64182024-02-27T21:46:30.050Z
64192024-02-27T21:46:30.050ZVerifying data now
64202024-02-27T21:46:32.940ZNo change after verify
64212024-02-27T21:46:32.940ZLoop: 29 Downstairs dump after verify (and repair):
64222024-02-27T21:46:32.955ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64232024-02-27T21:46:32.955Z 0 000-019 55 55 55 110 110 110 F F F
64242024-02-27T21:46:32.955Z 1 020-039 50 50 50 101 101 101 F F F
64252024-02-27T21:46:32.955Z 2 040-059 56 56 56 111 111 111 F F F
64262024-02-27T21:46:32.955Z 3 060-079 58 58 58 113 113 113 F F F
64272024-02-27T21:46:32.955Z 4 080-099 56 56 56 111 111 111 F F F
64282024-02-27T21:46:32.955Z 5 100-119 49 49 49 99 99 99 F F F
64292024-02-27T21:46:32.955Z 6 120-139 56 56 56 111 111 111 F F F
64302024-02-27T21:46:32.955Z 7 140-159 58 58 58 113 113 113 F F F
64312024-02-27T21:46:32.956Z 8 160-179 58 58 58 113 113 113 F F F
64322024-02-27T21:46:32.956Z 9 180-199 58 58 58 113 113 113 F F F
64332024-02-27T21:46:32.956Z 10 200-219 59 59 59 114 114 114 F F F
64342024-02-27T21:46:32.956Z 11 220-239 58 58 58 113 113 113 F F F
64352024-02-27T21:46:32.956Z 12 240-259 56 56 56 111 111 111 F F F
64362024-02-27T21:46:32.956Z 13 260-279 56 56 56 111 111 111 F F F
64372024-02-27T21:46:32.956Z 14 280-299 56 56 56 111 111 111 F F F
64382024-02-27T21:46:32.956Z 15 300-319 58 58 58 113 113 113 F F F
64392024-02-27T21:46:32.956Z 16 320-339 58 58 58 113 113 113 F F F
64402024-02-27T21:46:32.956Z 17 340-359 54 54 54 109 109 109 F F F
64412024-02-27T21:46:32.956Z 18 360-379 58 58 58 113 113 113 F F F
64422024-02-27T21:46:32.957Z 19 380-399 54 54 54 109 109 109 F F F
64432024-02-27T21:46:32.957Z 20 400-419 58 58 58 112 112 112 F F F
64442024-02-27T21:46:32.957Z 21 420-439 59 59 59 114 114 114 F F F
64452024-02-27T21:46:32.957Z 22 440-459 56 56 56 111 111 111 F F F
64462024-02-27T21:46:32.957Z 23 460-479 58 58 58 113 113 113 F F F
64472024-02-27T21:46:32.957Z 24 480-499 50 50 50 101 101 101 F F F
64482024-02-27T21:46:32.957Z 25 500-519 56 56 56 111 111 111 F F F
64492024-02-27T21:46:32.957Z 26 520-539 56 56 56 111 111 111 F F F
64502024-02-27T21:46:32.957Z 27 540-559 59 59 59 114 114 114 F F F
64512024-02-27T21:46:32.958Z 28 560-579 58 58 58 113 113 113 F F F
64522024-02-27T21:46:32.958Z 29 580-599 44 44 44 79 79 79 F F F
64532024-02-27T21:46:32.958ZMax gen: 59, Max flush: 114
64542024-02-27T21:46:32.958Z
64552024-02-27T21:46:32.958ZBegin loop 30
64562024-02-27T21:46:32.958ZDownstairs to restart: 1
64572024-02-27T21:46:32.972Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362032582Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381}
64582024-02-27T21:46:32.972Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362282253Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381}
64592024-02-27T21:46:32.973Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362300351Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381}
64602024-02-27T21:46:32.973Z{"msg":"Crucible 1372469c-2165-4118-9333-f596c11b0903 has session id: c530c1a3-b239-4474-99eb-addca2a618fd","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362314669Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64612024-02-27T21:46:32.973Z{"msg":"Upstairs opts: Upstairs UUID: 1372469c-2165-4118-9333-f596c11b0903, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362347415Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64622024-02-27T21:46:32.973Z{"msg":"Crucible stats registered with UUID: 1372469c-2165-4118-9333-f596c11b0903","v":0,"name":"crucible","Crucible runtime is spawned
64632024-02-27T21:46:32.973Zlevel":30,"time":"2024-02-27T21:46:31.362382611Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64642024-02-27T21:46:32.973Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362459602Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381}
64652024-02-27T21:46:32.973Z{"msg":"1372469c-2165-4118-9333-f596c11b0903 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362609844Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64662024-02-27T21:46:32.973Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362636951Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64672024-02-27T21:46:32.973Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362658069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64682024-02-27T21:46:32.973Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362674967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64692024-02-27T21:46:32.974Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362699254Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64702024-02-27T21:46:32.974Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.36273126Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64712024-02-27T21:46:32.974Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362758227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64722024-02-27T21:46:32.974Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362782984Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64732024-02-27T21:46:32.974Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362823729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64742024-02-27T21:46:32.974Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.362873453Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64752024-02-27T21:46:32.974Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.36298385Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64762024-02-27T21:46:32.974Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.363024075Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64772024-02-27T21:46:32.974Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.363217763Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64782024-02-27T21:46:32.987Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.363680098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64792024-02-27T21:46:32.987Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.363721683Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64802024-02-27T21:46:32.987Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.363807213Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64812024-02-27T21:46:32.988Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.363861087Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64822024-02-27T21:46:32.988Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.363904862Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64832024-02-27T21:46:32.988Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.363931799Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64842024-02-27T21:46:32.988Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.364188958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64852024-02-27T21:46:32.988Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.364565454Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64862024-02-27T21:46:32.988Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.364624247Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64872024-02-27T21:46:32.988Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.364692649Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64882024-02-27T21:46:32.988Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.364718166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64892024-02-27T21:46:32.988Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.364756301Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64902024-02-27T21:46:32.989Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365182421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64912024-02-27T21:46:32.989Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365218717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64922024-02-27T21:46:32.989Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365234535Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64932024-02-27T21:46:32.989Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365297348Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64942024-02-27T21:46:32.989Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365323584Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64952024-02-27T21:46:32.989Z{"msg":"[0]R flush_numbers[0..12]: [110, 101, 111, 113, 111, 99, 111, 113, 113, 113, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365338423Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64962024-02-27T21:46:32.989Z{"msg":"[0]R generation[0..12]: [55, 50, 56, 58, 56, 49, 56, 58, 58, 58, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.36536038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64972024-02-27T21:46:32.989Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365375978Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64982024-02-27T21:46:32.989Z{"msg":"[1]R flush_numbers[0..12]: [110, 101, 111, 113, 111, 99, 111, 113, 113, 113, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365397996Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
64992024-02-27T21:46:32.990Z{"msg":"[1]R generation[0..12]: [55, 50, 56, 58, 56, 49, 56, 58, 58, 58, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365412134Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65002024-02-27T21:46:32.990Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365439331Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65012024-02-27T21:46:32.990Z{"msg":"[2]R flush_numbers[0..12]: [110, 101, 111, 113, 111, 99, 111, 113, 113, 113, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365454999Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65022024-02-27T21:46:32.990Z{"msg":"[2]R generation[0..12]: [55, 50, 56, 58, 56, 49, 56, 58, 58, 58, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365469058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65032024-02-27T21:46:32.990Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365483696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,Wait for a query_work_queue command to finish before sending IO
65042024-02-27T21:46:32.990Z"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65052024-02-27T21:46:32.990Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365500564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65062024-02-27T21:46:32.990Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365522121Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65072024-02-27T21:46:32.990Z{"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.36553519Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65082024-02-27T21:46:32.991Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365560697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"":"downstairs","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65092024-02-27T21:46:32.991Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365574425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65102024-02-27T21:46:32.991Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365586513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"0","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65112024-02-27T21:46:32.991Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365607301Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"1","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65122024-02-27T21:46:32.992Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.36562102Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"client":"2","":"downstairs","session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65132024-02-27T21:46:32.992Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365634908Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65142024-02-27T21:46:32.992Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365647257Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65152024-02-27T21:46:32.992Z{"msg":"1372469c-2165-4118-9333-f596c11b0903 is now active with session: c530c1a3-b239-4474-99eb-addca2a618fd","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365660615Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65162024-02-27T21:46:32.992Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365672694Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381,"session_id":"c530c1a3-b239-4474-99eb-addca2a618fd"}
65172024-02-27T21:46:32.992Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:31.365693141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381}
65182024-02-27T21:46:32.993ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65192024-02-27T21:46:32.993ZLoading write count information from file "/tmp/test_repair_verify.data"
65202024-02-27T21:46:32.993ZRun Repair workload
65212024-02-27T21:46:32.993Z01/30 Write block 63 len 12288 data: 4 3 5
65222024-02-27T21:46:32.993Z02/30 Read block 66 len 20480
65232024-02-27T21:46:32.994Z03/30 Read block 195 len 12288
65242024-02-27T21:46:32.994Z04/30 Read block 537 len 4096
65252024-02-27T21:46:32.994Z05/30 Read block 221 len 32768
65262024-02-27T21:46:32.994Z06/30 Read block 407 len 12288
65272024-02-27T21:46:32.994Z07/30 Flush
65282024-02-27T21:46:32.994Z08/30 Write block 115 len 40960 data: 3 3 5 5 4 5 6 6 6 5
65292024-02-27T21:46:32.995Z09/30 Read block 466 len 36864
65302024-02-27T21:46:32.995Z10/30 Read block 79 len 12288
65312024-02-27T21:46:32.995Z11/30 Write block 445 len 8192 data: 5 6
65322024-02-27T21:46:32.995Z12/30 Write block 31 len 16384 data: 6 5 5 7
65332024-02-27T21:46:33.433Z13/30 Read block 240 len 8192
65342024-02-27T21:46:33.433Z14/30 Read block 364 len 20480
65352024-02-27T21:46:33.433Z15/30 Read block 509 len 32768
65362024-02-27T21:46:33.433Z16/30 Flush
65372024-02-27T21:46:33.434Z17/30 Write block 245 len 36864 data: 6 6 7 6 6 5 4 5 5
65382024-02-27T21:46:33.434Z18/30 Write block 582 len 12288 data: 5 4 4
65392024-02-27T21:46:33.434Z19/30 Write block 430 len 4096 data: 5
65402024-02-27T21:46:33.434Z20/30 Write block 80 len 8192 data: 4 4
65412024-02-27T21:46:33.434Z21/30 Read block 311 len 28672
65422024-02-27T21:46:33.434Z22/30 Read block 107 len 32768
65432024-02-27T21:46:33.435Z23/30 Read block 292 len 12288
65442024-02-27T21:46:33.435Z24/30 Write block 475 len 8192 data: 3 2
65452024-02-27T21:46:33.435Z25/30 Read block 192 len 12288
65462024-02-27T21:46:33.435Z26/30 Read block 158 len 40960
65472024-02-27T21:46:33.435Z27/30 Read block 372 len 4096
65482024-02-27T21:46:33.435Z28/30 Write block 592 len 16384 data: 3 3 3 3
65492024-02-27T21:46:33.435Z29/30 Write block 548 len 24576 data: 6 6 5 7 5 6
65502024-02-27T21:46:33.436Z30/30 Write block 64 len 36864 data: 4 6 5 5 6 5 5 5 6
65512024-02-27T21:46:33.436Z----------------------------------------------------------------
65522024-02-27T21:46:33.436Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65532024-02-27T21:46:33.436ZStates: Active Active Active
65542024-02-27T21:46:33.436ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65552024-02-27T21:46:33.437Z 1 Acked 1000 Write 3 Done Sent Done false
65562024-02-27T21:46:33.437Z 2 Acked 1001 Read 5 Done Sent Done false
65572024-02-27T21:46:33.437Z 3 Acked 1002 Read 3 Done Sent Done false
65582024-02-27T21:46:33.437Z 4 Acked 1003 Read 1 Done Sent Done false
65592024-02-27T21:46:33.437Z 5 Acked 1004 Read 8 Done Sent Done false
65602024-02-27T21:46:33.437Z 6 Acked 1005 Read 3 Done Sent Done false
65612024-02-27T21:46:33.438Z 7 Acked 1006 Flush 0 Done Sent Done false
65622024-02-27T21:46:33.438Z 8 Acked 1007 Write 10 Done Sent Done false
65632024-02-27T21:46:33.438Z 9 Acked 1008 Read 9 Done Sent Done false
65642024-02-27T21:46:33.438Z 10 Acked 1009 Read 3 Done Sent Done false
65652024-02-27T21:46:33.438Z 11 Acked 1010 Write 2 Done Sent Done false
65662024-02-27T21:46:33.438Z 12 Acked 1011 Write 4 Done Sent Done false
65672024-02-27T21:46:33.438Z 13 Acked 1012 Read 2 Done Sent Done false
65682024-02-27T21:46:33.439Z 14 Acked 1013 Read 5 Done Sent Done false
65692024-02-27T21:46:33.439Z 15 Acked 1014 Read 8 Done Sent Done false
65702024-02-27T21:46:33.439Z 16 Acked 1015 Flush 0 Done Sent Done false
65712024-02-27T21:46:33.439Z 17 Acked 1016 Write 9 Done Sent Done false
65722024-02-27T21:46:33.439Z 18 Acked 1017 Write 3 Done Sent Done false
65732024-02-27T21:46:33.439Z 19 Acked 1018 Write 1 Done Sent Done false
65742024-02-27T21:46:33.439Z 20 Acked 1019 Write 2 Done Sent Done false
65752024-02-27T21:46:33.439Z 21 Acked 1020 Read 7 Done Sent Done false
65762024-02-27T21:46:33.439Z 22 Acked 1021 Read 8 Done Sent Done false
65772024-02-27T21:46:33.439Z 23 Acked 1022 Read 3 Done Sent Done false
65782024-02-27T21:46:33.440Z 24 Acked 1023 Write 2 Done Sent Done false
65792024-02-27T21:46:33.440Z 25 Acked 1024 Read 3 Done Sent Done false
65802024-02-27T21:46:33.440Z 26 Acked 1025 Read 10 Done Sent Done false
65812024-02-27T21:46:33.440Z 27 Acked 1026 Read 1 Done Sent Done false
65822024-02-27T21:46:33.440Z 28 Acked 1027 Write 4 Done Sent Done false
65832024-02-27T21:46:33.440Z 29 Acked 1028 Write 6 Done Sent Done false
65842024-02-27T21:46:33.441Z 30 Acked 1029 Write 9 Sent Sent Sent false
65852024-02-27T21:46:33.441Z STATES DS:0 DS:1 DS:2 TOTAL
65862024-02-27T21:46:33.441Z New 0 0 0 0
65872024-02-27T21:46:33.441Z Sent 1 30 1 32
65882024-02-27T21:46:33.441Z Done 29 0 29 58
65892024-02-27T21:46:33.441Z Skipped 0 0 0 0
65902024-02-27T21:46:33.442Z Error 0 0 0 0
65912024-02-27T21:46:33.442ZLast Flush: 1015 0 1015
65922024-02-27T21:46:33.442ZDownstairs last five completed:
65932024-02-27T21:46:33.442ZUpstairs last five completed: 30 29 28 27 26
65942024-02-27T21:46:33.442Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:31.393189052Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1381}
65952024-02-27T21:46:33.443ZWrote out file "/tmp/test_repair_verify.data"
65962024-02-27T21:46:33.443Z
65972024-02-27T21:46:33.443ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
65982024-02-27T21:46:33.473ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65992024-02-27T21:46:33.473Z 0 000-019 55 55 55 110 110 110 F F F
66002024-02-27T21:46:33.473Z 1 020-039 60 50 60 116 101 116 F F F <---
66012024-02-27T21:46:33.473Z 2 040-059 56 56 56 111 111 111 F F F
66022024-02-27T21:46:33.473Z 3 060-079 60 58 60 115 113 115 T F T <---
66032024-02-27T21:46:33.473Z 4 080-099 56 56 56 111 111 111 T F T <---
66042024-02-27T21:46:33.473Z 5 100-119 60 49 60 116 99 116 F F F <---
66052024-02-27T21:46:33.473Z 6 120-139 60 56 60 116 111 116 F F F <---
66062024-02-27T21:46:33.473Z 7 140-159 58 58 58 113 113 113 F F F
66072024-02-27T21:46:33.473Z 8 160-179 58 58 58 113 113 113 F F F
66082024-02-27T21:46:33.474Z 9 180-199 58 58 58 113 113 113 F F F
66092024-02-27T21:46:33.474Z 10 200-219 59 59 59 114 114 114 F F F
66102024-02-27T21:46:33.474Z 11 220-239 58 58 58 113 113 113 F F F
66112024-02-27T21:46:33.474Z 12 240-259 56 56 56 111 111 111 T F T <---
66122024-02-27T21:46:33.474Z 13 260-279 56 56 56 111 111 111 F F F
66132024-02-27T21:46:33.474Z 14 280-299 56 56 56 111 111 111 F F F
66142024-02-27T21:46:33.474Z 15 300-319 58 58 58 113 113 113 F F F
66152024-02-27T21:46:33.474Z 16 320-339 58 58 58 113 113 113 F F F
66162024-02-27T21:46:33.474Z 17 340-359 54 54 54 109 109 109 F F F
66172024-02-27T21:46:33.474Z 18 360-379 58 58 58 113 113 113 F F F
66182024-02-27T21:46:33.474Z 19 380-399 54 54 54 109 109 109 F F F
66192024-02-27T21:46:33.475Z 20 400-419 58 58 58 112 112 112 F F F
66202024-02-27T21:46:33.475Z 21 420-439 59 59 59 114 114 114 T F T <---
66212024-02-27T21:46:33.475Z 22 440-459 60 56 60 116 111 116 F F F <---
66222024-02-27T21:46:33.475Z 23 460-479 58 58 58 113 113 113 T F T <---
66232024-02-27T21:46:33.475Z 24 480-499 50 50 50 101 101 101 F F F
66242024-02-27T21:46:33.475Z 25 500-519 56 56 56 111 111 111 F F F
66252024-02-27T21:46:33.475Z 26 520-539 56 56 56 111 111 111 F F F
66262024-02-27T21:46:33.475Z 27 540-559 59 59 59 114 114 114 T F T <---
66272024-02-27T21:46:33.475Z 28 560-579 58 58 58 113 113 113 F F F
66282024-02-27T21:46:33.475Z 29 580-599 44 44 44 79 79 79 T F T <---
66292024-02-27T21:46:33.476ZMax gen: 60, Max flush: 116
66302024-02-27T21:46:33.476ZError: Difference in extent metadata found!
66312024-02-27T21:46:33.476ZA Difference in extent metadata is expected here
66322024-02-27T21:46:33.476ZOn loop 30
66332024-02-27T21:46:33.476Z
66342024-02-27T21:46:33.476ZVerifying data now
66352024-02-27T21:46:37.587ZNo change after verify
66362024-02-27T21:46:37.587ZLoop: 30 Downstairs dump after verify (and repair):
66372024-02-27T21:46:37.601ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66382024-02-27T21:46:37.601Z 0 000-019 55 55 55 110 110 110 F F F
66392024-02-27T21:46:37.601Z 1 020-039 60 60 60 116 116 116 F F F
66402024-02-27T21:46:37.601Z 2 040-059 56 56 56 111 111 111 F F F
66412024-02-27T21:46:37.601Z 3 060-079 61 61 61 117 117 117 F F F
66422024-02-27T21:46:37.601Z 4 080-099 61 61 61 117 117 117 F F F
66432024-02-27T21:46:37.601Z 5 100-119 60 60 60 116 116 116 F F F
66442024-02-27T21:46:37.601Z 6 120-139 60 60 60 116 116 116 F F F
66452024-02-27T21:46:37.608Z 7 140-159 58 58 58 113 113 113 F F F
66462024-02-27T21:46:37.608Z 8 160-179 58 58 58 113 113 113 F F F
66472024-02-27T21:46:37.608Z 9 180-199 58 58 58 113 113 113 F F F
66482024-02-27T21:46:37.608Z 10 200-219 59 59 59 114 114 114 F F F
66492024-02-27T21:46:37.608Z 11 220-239 58 58 58 113 113 113 F F F
66502024-02-27T21:46:37.608Z 12 240-259 61 61 61 117 117 117 F F F
66512024-02-27T21:46:37.608Z 13 260-279 56 56 56 111 111 111 F F F
66522024-02-27T21:46:37.608Z 14 280-299 56 56 56 111 111 111 F F F
66532024-02-27T21:46:37.608Z 15 300-319 58 58 58 113 113 113 F F F
66542024-02-27T21:46:37.608Z 16 320-339 58 58 58 113 113 113 F F F
66552024-02-27T21:46:37.609Z 17 340-359 54 54 54 109 109 109 F F F
66562024-02-27T21:46:37.609Z 18 360-379 58 58 58 113 113 113 F F F
66572024-02-27T21:46:37.609Z 19 380-399 54 54 54 109 109 109 F F F
66582024-02-27T21:46:37.609Z 20 400-419 58 58 58 112 112 112 F F F
66592024-02-27T21:46:37.609Z 21 420-439 61 61 61 117 117 117 F F F
66602024-02-27T21:46:37.609Z 22 440-459 60 60 60 116 116 116 F F F
66612024-02-27T21:46:37.609Z 23 460-479 61 61 61 117 117 117 F F F
66622024-02-27T21:46:37.609Z 24 480-499 50 50 50 101 101 101 F F F
66632024-02-27T21:46:37.609Z 25 500-519 56 56 56 111 111 111 F F F
66642024-02-27T21:46:37.609Z 26 520-539 56 56 56 111 111 111 F F F
66652024-02-27T21:46:37.609Z 27 540-559 61 61 61 117 117 117 F F F
66662024-02-27T21:46:37.609Z 28 560-579 58 58 58 113 113 113 F F F
66672024-02-27T21:46:37.609Z 29 580-599 61 61 61 117 117 117 F F F
66682024-02-27T21:46:37.609ZMax gen: 61, Max flush: 117
66692024-02-27T21:46:37.609Z
66702024-02-27T21:46:37.609ZBegin loop 31
66712024-02-27T21:46:37.609ZDownstairs to restart: 0
66722024-02-27T21:46:37.649Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020040735Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389}
66732024-02-27T21:46:37.649Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020398353Z","Crucible runtime is spawned
66742024-02-27T21:46:37.649Zhostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389}
66752024-02-27T21:46:37.688Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020450546Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389}
66762024-02-27T21:46:37.688Z{"msg":"Crucible 643e07d2-ccfd-4ead-883e-e79047a37487 has session id: b1b83ea4-3e64-41cb-a5cb-2d34b21273f1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020492872Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66772024-02-27T21:46:37.688Z{"msg":"Upstairs opts: Upstairs UUID: 643e07d2-ccfd-4ead-883e-e79047a37487, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020540826Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66782024-02-27T21:46:37.689Z{"msg":"Crucible stats registered with UUID: 643e07d2-ccfd-4ead-883e-e79047a37487","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020583661Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66792024-02-27T21:46:37.689Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020613587Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389}
66802024-02-27T21:46:37.689Z{"msg":"643e07d2-ccfd-4ead-883e-e79047a37487 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020650183Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66812024-02-27T21:46:37.690Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.02067824Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66822024-02-27T21:46:37.690Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020735523Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66832024-02-27T21:46:37.690Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020769439Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66842024-02-27T21:46:37.690Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020842291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66852024-02-27T21:46:37.690Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020893434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66862024-02-27T21:46:37.690Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.020990213Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66872024-02-27T21:46:37.691Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021029098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66882024-02-27T21:46:37.691Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021059335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66892024-02-27T21:46:37.691Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021084442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66902024-02-27T21:46:37.691Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021116438Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66912024-02-27T21:46:37.691Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021132766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66922024-02-27T21:46:37.692Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021157473Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66932024-02-27T21:46:37.692Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021645176Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66942024-02-27T21:46:37.692Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021730916Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66952024-02-27T21:46:37.692Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021812046Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66962024-02-27T21:46:37.692Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021857001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66972024-02-27T21:46:37.692Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021881468Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66982024-02-27T21:46:37.693Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.021911634Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
66992024-02-27T21:46:37.693Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022240956Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67002024-02-27T21:46:37.693Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022310547Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67012024-02-27T21:46:37.693Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022346223Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67022024-02-27T21:46:37.693Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022785311Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67032024-02-27T21:46:37.694Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022832506Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67042024-02-27T21:46:37.694Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022855933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67052024-02-27T21:46:37.694Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022894769Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67062024-02-27T21:46:37.694Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022921415Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67072024-02-27T21:46:37.694Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022948952Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67082024-02-27T21:46:37.694Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.02297154Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67092024-02-27T21:46:37.695Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.022999886Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67102024-02-27T21:46:37.695Z{"msg":"[0]R flush_numbers[0..12]: [110, 116, 111, 117, 117, 116, 116, 113, 113, 113, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023025313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67112024-02-27T21:46:37.695Z{"msg":"[0]R generation[0..12]: [55, 60, 56, 61, 61, 60, 60, 58, 58, 58, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.02305839Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67122024-02-27T21:46:37.695Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023085656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67132024-02-27T21:46:37.695Z{"msg":"[1]R flush_numbers[0..12]: [110, 116, 111, 117, 117, 116, 116, 113, 113, 113, 114, 113]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
67142024-02-27T21:46:37.695Z,"time":"2024-02-27T21:46:36.023118562Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67152024-02-27T21:46:37.695Z{"msg":"[1]R generation[0..12]: [55, 60, 56, 61, 61, 60, 60, 58, 58, 58, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023147279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67162024-02-27T21:46:37.695Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023188974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67172024-02-27T21:46:37.696Z{"msg":"[2]R flush_numbers[0..12]: [110, 116, 111, 117, 117, 116, 116, 113, 113, 113, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023212411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67182024-02-27T21:46:37.696Z{"msg":"[2]R generation[0..12]: [55, 60, 56, 61, 61, 60, 60, 58, 58, 58, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023249087Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67192024-02-27T21:46:37.696Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023274494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67202024-02-27T21:46:37.696Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.02330887Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67212024-02-27T21:46:37.696Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023334557Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67222024-02-27T21:46:37.697Z{"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023369033Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67232024-02-27T21:46:37.697Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023404199Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"":"downstairs","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67242024-02-27T21:46:37.697Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023444354Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67252024-02-27T21:46:37.697Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.02347919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"0","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67262024-02-27T21:46:37.697Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023518685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"1","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67272024-02-27T21:46:37.698Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67282024-02-27T21:46:37.698Z,"time":"2024-02-27T21:46:36.02356184Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"client":"2","":"downstairs","session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67292024-02-27T21:46:37.698Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023611334Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67302024-02-27T21:46:37.698Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023635861Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67312024-02-27T21:46:37.698Z{"msg":"643e07d2-ccfd-4ead-883e-e79047a37487 is now active with session: b1b83ea4-3e64-41cb-a5cb-2d34b21273f1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023669567Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67322024-02-27T21:46:37.698Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.023699494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389,"session_id":"b1b83ea4-3e64-41cb-a5cb-2d34b21273f1"}
67332024-02-27T21:46:37.698Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:36.02372961Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389}
67342024-02-27T21:46:37.698ZLoading write count information from file "/tmp/test_repair_verify.data"
67352024-02-27T21:46:37.699ZRun Repair workload
67362024-02-27T21:46:37.699Z01/30 Write block 504 len 32768 data: 4 5 5 5 3 6 7 11
67372024-02-27T21:46:37.699Z02/30 Flush
67382024-02-27T21:46:37.699Z03/30 Write block 192 len 4096 data: 7
67392024-02-27T21:46:37.699Z04/30 Read block 445 len 4096
67402024-02-27T21:46:37.699Z05/30 Flush
67412024-02-27T21:46:37.699Z06/30 Write block 56 len 8192 data: 9 9
67422024-02-27T21:46:37.700Z07/30 Read block 41 len 28672
67432024-02-27T21:46:37.700Z08/30 Write block 476 len 16384 data: 3 3 4 5
67442024-02-27T21:46:37.700Z09/30 Write block 349 len 4096 data: 7
67452024-02-27T21:46:37.700Z10/30 Read block 13 len 20480
67462024-02-27T21:46:37.700Z11/30 Read block 114 len 36864
67472024-02-27T21:46:37.701Z12/30 Write block 255 len 24576 data: 5 5 3 5 5 6
67482024-02-27T21:46:37.701Z13/30 Read block 46 len 4096
67492024-02-27T21:46:37.701Z14/30 Write block 510 len 20480 data: 8 12 11 9 7
67502024-02-27T21:46:37.701Z15/30 Read block 239 len 12288
67512024-02-27T21:46:37.701Z16/30 Write block 24 len 4096 data: 8
67522024-02-27T21:46:37.702Z17/30 Read block 98 len 32768
67532024-02-27T21:46:37.702Z18/30 Read block 103 len 36864
67542024-02-27T21:46:37.702Z19/30 Write block 114 len 28672 data: 4 4 4 6 6 5 6
67552024-02-27T21:46:37.702Z20/30 Write block 31 len 24576 data: 7 6 6 8 6 6
67562024-02-27T21:46:37.702Z21/30 Read block 188 len 20480
67572024-02-27T21:46:37.702Z22/30 Write block 514 len 16384 data: 8 6 7 7
67582024-02-27T21:46:37.702Z23/30 Write block 255 len 16384 data: 6 6 4 6
67592024-02-27T21:46:37.702Z24/30 Flush
67602024-02-27T21:46:37.702Z25/30 Write block 119 len 40960 data: 6 7 7 7 7 6 5 5 5 6
67612024-02-27T21:46:37.702Z26/30 Write block 453 len 4096 data: 7
67622024-02-27T21:46:37.702Z27/30 Write block 86 len 24576 data: 4 5 7 8 6 5
67632024-02-27T21:46:37.703Z28/30 Read block 38 len 12288
67642024-02-27T21:46:37.703Z29/30 Read block 8 len 28672
67652024-02-27T21:46:37.703Z30/30 Write block 252 len 36864 data: 6 6 4 7 7 5 7 6 7
67662024-02-27T21:46:37.703Z----------------------------------------------------------------
67672024-02-27T21:46:37.703Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:28
67682024-02-27T21:46:37.703ZStates: Active Active Active
67692024-02-27T21:46:37.703ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67702024-02-27T21:46:37.703Z 3 Acked 1002 Write 1 Done Done Done false
67712024-02-27T21:46:37.703Z 4 Acked 1003 Read 1 Done Done Done false
67722024-02-27T21:46:37.704Z 5 Acked 1004 Flush 0 Sent Done Done false
67732024-02-27T21:46:37.704Z 6 Acked 1005 Write 2 Sent Done Done false
67742024-02-27T21:46:37.704Z 7 Acked 1006 Read 7 Sent Done Done false
67752024-02-27T21:46:37.711Z 8 Acked 1007 Write 4 Sent Done Done false
67762024-02-27T21:46:37.711Z 9 Acked 1008 Write 1 Sent Done Done false
67772024-02-27T21:46:37.712Z 10 Acked 1009 Read 5 Sent Done Done false
67782024-02-27T21:46:37.712Z 11 Acked 1010 Read 9 Sent Done Done false
67792024-02-27T21:46:37.712Z 12 Acked 1011 Write 6 Sent Done Done false
67802024-02-27T21:46:37.712Z 13 Acked 1012 Read 1 Sent Done Done false
67812024-02-27T21:46:37.712Z 14 Acked 1013 Write 5 Sent Done Done false
67822024-02-27T21:46:37.712Z 15 Acked 1014 Read 3 Sent Done Done false
67832024-02-27T21:46:37.714Z 16 Acked 1015 Write 1 Sent Done Done false
67842024-02-27T21:46:37.714Z 17 Acked 1016 Read 8 Sent Done Done false
67852024-02-27T21:46:37.714Z 18 Acked 1017 Read 9 Sent Done Done false
67862024-02-27T21:46:37.714Z 19 Acked 1018 Write 7 Sent Done Done false
67872024-02-27T21:46:37.714Z 20 Acked 1019 Write 6 Sent Done Done false
67882024-02-27T21:46:37.714Z 21 Acked 1020 Read 5 Sent Done Done false
67892024-02-27T21:46:37.714Z 22 Acked 1021 Write 4 Sent Done Done false
67902024-02-27T21:46:37.714Z 23 Acked 1022 Write 4 Sent Done Done false
67912024-02-27T21:46:37.714Z 24 Acked 1023 Flush 0 Sent Done Done false
67922024-02-27T21:46:37.714Z 25 Acked 1024 Write 10 Sent Done Done false
67932024-02-27T21:46:37.715Z 26 Acked 1025 Write 1 Sent Done Done false
67942024-02-27T21:46:37.715Z 27 Acked 1026 Write 6 Sent Done Done false
67952024-02-27T21:46:37.715Z 28 Acked 1027 Read 3 Sent Done Done false
67962024-02-27T21:46:37.715Z 29 Acked 1028 Read 7 Sent Done Done false
67972024-02-27T21:46:37.715Z 30 Acked 1029 Write 9 Sent Sent Sent false
67982024-02-27T21:46:37.715Z STATES DS:0 DS:1 DS:2 TOTAL
67992024-02-27T21:46:37.715Z New 0 0 0 0
68002024-02-27T21:46:37.715Z Sent 26 1 1 28
68012024-02-27T21:46:37.715Z Done 2 27 27 56
68022024-02-27T21:46:37.715Z Skipped 0 0 0 0
68032024-02-27T21:46:37.715Z Error 0 0 0 0
68042024-02-27T21:46:37.715ZLast Flush: 1001 1023 1023
68052024-02-27T21:46:37.716ZDownstairs last five completed: 1001 1000
68062024-02-27T21:46:37.716ZUpstairs last five completed: 30 29 28 27 26
68072024-02-27T21:46:37.716Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:36.052799956Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1389}
68082024-02-27T21:46:37.716ZWrote out file "/tmp/test_repair_verify.data"
68092024-02-27T21:46:37.716Z
68102024-02-27T21:46:37.716ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68112024-02-27T21:46:37.716ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68122024-02-27T21:46:37.716Z 0 000-019 55 55 55 110 110 110 F F F
68132024-02-27T21:46:37.716Z 1 020-039 60 62 62 116 120 120 F F F <---
68142024-02-27T21:46:37.716Z 2 040-059 56 62 62 111 120 120 F F F <---
68152024-02-27T21:46:37.716Z 3 060-079 61 61 61 117 117 117 F F F
68162024-02-27T21:46:37.716Z 4 080-099 61 61 61 117 117 117 F T T <---
68172024-02-27T21:46:37.716Z 5 100-119 60 62 62 116 120 120 F T T <---
68182024-02-27T21:46:37.717Z 6 120-139 60 62 62 116 120 120 F T T <---
68192024-02-27T21:46:37.717Z 7 140-159 58 58 58 113 113 113 F F F
68202024-02-27T21:46:37.717Z 8 160-179 58 58 58 113 113 113 F F F
68212024-02-27T21:46:37.717Z 9 180-199 58 62 62 113 119 119 T F F <---
68222024-02-27T21:46:37.717Z 10 200-219 59 59 59 114 114 114 F F F
68232024-02-27T21:46:37.717Z 11 220-239 58 58 58 113 113 113 F F F
68242024-02-27T21:46:37.717Z 12 240-259 61 62 62 117 120 120 F T T <---
68252024-02-27T21:46:37.717Z 13 260-279 56 62 62 111 120 120 F T T <---
68262024-02-27T21:46:37.717Z 14 280-299 56 56 56 111 111 111 F F F
68272024-02-27T21:46:37.717Z 15 300-319 58 58 58 113 113 113 F F F
68282024-02-27T21:46:37.717Z 16 320-339 58 58 58 113 113 113 F F F
68292024-02-27T21:46:37.718Z 17 340-359 54 62 62 109 120 120 F F F <---
68302024-02-27T21:46:37.718Z 18 360-379 58 58 58 113 113 113 F F F
68312024-02-27T21:46:37.718Z 19 380-399 54 54 54 109 109 109 F F F
68322024-02-27T21:46:37.718Z 20 400-419 58 58 58 112 112 112 F F F
68332024-02-27T21:46:37.718Z 21 420-439 61 61 61 117 117 117 F F F
68342024-02-27T21:46:37.718Z 22 440-459 60 60 60 116 116 116 F T T <---
68352024-02-27T21:46:37.718Z 23 460-479 61 62 62 117 120 120 F F F <---
68362024-02-27T21:46:37.718Z 24 480-499 50 50 50 101 101 101 F F F
68372024-02-27T21:46:37.718Z 25 500-519 62 62 62 118 120 120 F F F <---
68382024-02-27T21:46:37.718Z 26 520-539 56 56 56 111 111 111 F F F
68392024-02-27T21:46:37.718Z 27 540-559 61 61 61 117 117 117 F F F
68402024-02-27T21:46:37.718Z 28 560-579 58 58 58 113 113 113 F F F
68412024-02-27T21:46:37.718Z 29 580-599 61 61 61 117 117 117 F F F
68422024-02-27T21:46:37.719ZMax gen: 62, Max flush: 120
68432024-02-27T21:46:37.719ZError: Difference in extent metadata found!
68442024-02-27T21:46:37.719ZA Difference in extent metadata is expected here
68452024-02-27T21:46:37.719ZOn loop 31
68462024-02-27T21:46:37.719Z
68472024-02-27T21:46:37.719ZVerifying data now
68482024-02-27T21:46:42.292ZNo change after verify
68492024-02-27T21:46:42.292ZLoop: 31 Downstairs dump after verify (and repair):
68502024-02-27T21:46:42.307ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68512024-02-27T21:46:42.307Z 0 000-019 55 55 55 110 110 110 F F F
68522024-02-27T21:46:42.312Z 1 020-039 62 62 62 120 120 120 F F F
68532024-02-27T21:46:42.313Z 2 040-059 62 62 62 120 120 120 F F F
68542024-02-27T21:46:42.313Z 3 060-079 61 61 61 117 117 117 F F F
68552024-02-27T21:46:42.313Z 4 080-099 63 63 63 121 121 121 F F F
68562024-02-27T21:46:42.313Z 5 100-119 63 63 63 121 121 121 F F F
68572024-02-27T21:46:42.313Z 6 120-139 63 63 63 121 121 121 F F F
68582024-02-27T21:46:42.313Z 7 140-159 58 58 58 113 113 113 F F F
68592024-02-27T21:46:42.313Z 8 160-179 58 58 58 113 113 113 F F F
68602024-02-27T21:46:42.314Z 9 180-199 62 62 62 119 119 119 F F F
68612024-02-27T21:46:42.314Z 10 200-219 59 59 59 114 114 114 F F F
68622024-02-27T21:46:42.314Z 11 220-239 58 58 58 113 113 113 F F F
68632024-02-27T21:46:42.314Z 12 240-259 63 63 63 121 121 121 F F F
68642024-02-27T21:46:42.314Z 13 260-279 63 63 63 121 121 121 F F F
68652024-02-27T21:46:42.314Z 14 280-299 56 56 56 111 111 111 F F F
68662024-02-27T21:46:42.314Z 15 300-319 58 58 58 113 113 113 F F F
68672024-02-27T21:46:42.315Z 16 320-339 58 58 58 113 113 113 F F F
68682024-02-27T21:46:42.315Z 17 340-359 62 62 62 120 120 120 F F F
68692024-02-27T21:46:42.315Z 18 360-379 58 58 58 113 113 113 F F F
68702024-02-27T21:46:42.315Z 19 380-399 54 54 54 109 109 109 F F F
68712024-02-27T21:46:42.315Z 20 400-419 58 58 58 112 112 112 F F F
68722024-02-27T21:46:42.315Z 21 420-439 61 61 61 117 117 117 F F F
68732024-02-27T21:46:42.315Z 22 440-459 63 63 63 121 121 121 F F F
68742024-02-27T21:46:42.315Z 23 460-479 62 62 62 120 120 120 F F F
68752024-02-27T21:46:42.316Z 24 480-499 50 50 50 101 101 101 F F F
68762024-02-27T21:46:42.316Z 25 500-519 62 62 62 120 120 120 F F F
68772024-02-27T21:46:42.316Z 26 520-539 56 56 56 111 111 111 F F F
68782024-02-27T21:46:42.316Z 27 540-559 61 61 61 117 117 117 F F F
68792024-02-27T21:46:42.316Z 28 560-579 58 58 58 113 113 113 F F F
68802024-02-27T21:46:42.316Z 29 580-599 61 61 61 117 117 117 F F F
68812024-02-27T21:46:42.316ZMax gen: 63, Max flush: 121
68822024-02-27T21:46:42.316Z
68832024-02-27T21:46:42.316ZBegin loop 32
68842024-02-27T21:46:42.316ZDownstairs to restart: 2
68852024-02-27T21:46:42.332Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.724245355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397}
68862024-02-27T21:46:42.332Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.724509304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397}
68872024-02-27T21:46:42.332Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.724531341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397}
68882024-02-27T21:46:42.341Z{"msg":"Crucible 303afbfb-f1b5-43e6-84dd-311176fcb0fc has session id: 2fb635a3-1d62-49f2-9750-b7dcf03f0a22","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.724565137Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397Crucible runtime is spawned
68892024-02-27T21:46:42.341Z,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
68902024-02-27T21:46:42.341Z{"msg":"Upstairs opts: Upstairs UUID: 303afbfb-f1b5-43e6-84dd-311176fcb0fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.724608992Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
68912024-02-27T21:46:42.341Z{"msg":"Crucible stats registered with UUID: 303afbfb-f1b5-43e6-84dd-311176fcb0fc","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.724638349Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
68922024-02-27T21:46:42.342Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.724661406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397}
68932024-02-27T21:46:42.342Z{"msg":"303afbfb-f1b5-43e6-84dd-311176fcb0fc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.724913327Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
68942024-02-27T21:46:42.342Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.724973589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
68952024-02-27T21:46:42.342Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725006116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
68962024-02-27T21:46:42.342Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725034952Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
68972024-02-27T21:46:42.342Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725081587Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
68982024-02-27T21:46:42.342Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725107993Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
68992024-02-27T21:46:42.343Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725153078Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69002024-02-27T21:46:42.343Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725203842Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69012024-02-27T21:46:42.343Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725363294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69022024-02-27T21:46:42.343Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725414317Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69032024-02-27T21:46:42.343Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725459182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69042024-02-27T21:46:42.343Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725500847Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69052024-02-27T21:46:42.343Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.725545902Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69062024-02-27T21:46:42.343Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.726013277Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69072024-02-27T21:46:42.344Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.726056132Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69082024-02-27T21:46:42.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.726085178Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69092024-02-27T21:46:42.344Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.726099457Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69102024-02-27T21:46:42.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.726137172Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69112024-02-27T21:46:42.344Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.726204455Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69122024-02-27T21:46:42.344Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.72666536Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69132024-02-27T21:46:42.344Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.726703926Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69142024-02-27T21:46:42.345Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.726723323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69152024-02-27T21:46:42.345Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727239173Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69162024-02-27T21:46:42.345Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727298506Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69172024-02-27T21:46:42.345Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727330022Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69182024-02-27T21:46:42.345Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.72734412Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69192024-02-27T21:46:42.345Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727365528Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69202024-02-27T21:46:42.345Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727379326Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69212024-02-27T21:46:42.345Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727398554Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69222024-02-27T21:46:42.346Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727412522Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69232024-02-27T21:46:42.346Z{"msg":"[0]R flush_numbers[0..12]: [110, 120, 120, 117, 121, 121, 121, 113, 113, 119, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.72743327Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69242024-02-27T21:46:42.346Z{"msg":"[0]R generation[0..12]: [55, 62, 62, 61, 63, 63, 63, 58, 58, 62, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727447488Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69252024-02-27T21:46:42.346Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727461796Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69262024-02-27T21:46:42.346Z{"msg":"[1]R flush_numbers[0..12]: [110, 120, 120, 117, 121, 121, 121, 113, 113, 119, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727476694Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69272024-02-27T21:46:42.346Z{"msg":"[1]R generation[0..12]: [55, 62, 62, 61, 63, 63, 63, 58, 58, 62, 59, 58]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
69282024-02-27T21:46:42.346Z2024-02-27T21:46:40.727490283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69292024-02-27T21:46:42.346Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727507901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69302024-02-27T21:46:42.346Z{"msg":"[2]R flush_numbers[0..12]: [110, 120, 120, 117, 121, 121, 121, 113, 113, 119, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727522799Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69312024-02-27T21:46:42.347Z{"msg":"[2]R generation[0..12]: [55, 62, 62, 61, 63, 63, 63, 58, 58, 62, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727536467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69322024-02-27T21:46:42.347Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727557425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69332024-02-27T21:46:42.347Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727570843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69342024-02-27T21:46:42.347Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727590861Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69352024-02-27T21:46:42.347Z{"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727604019Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69362024-02-27T21:46:42.347Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727623687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"":"downstairs","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69372024-02-27T21:46:42.347Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727637406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69382024-02-27T21:46:42.347Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727649944Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"0","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69392024-02-27T21:46:42.348Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727669642Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"1","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69402024-02-27T21:46:42.348Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.72768287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"client":"2","":"downstairs","session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69412024-02-27T21:46:42.348Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727696569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69422024-02-27T21:46:42.348Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727708937Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69432024-02-27T21:46:42.348Z{"msg":"303afbfb-f1b5-43e6-84dd-311176fcb0fc is now active with session: 2fb635a3-1d62-49f2-9750-b7dcf03f0a22","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727721776Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69442024-02-27T21:46:42.348Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727741353Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397,"session_id":"2fb635a3-1d62-49f2-9750-b7dcf03f0a22"}
69452024-02-27T21:46:42.348Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:40.727753772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397}
69462024-02-27T21:46:42.349ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69472024-02-27T21:46:42.349ZLoading write count information from file "/tmp/test_repair_verify.data"
69482024-02-27T21:46:42.349ZRun Repair workload
69492024-02-27T21:46:42.349Z01/30 Write block 181 len 32768 data: 8 8 8 7 4 6 6 8
69502024-02-27T21:46:42.349Z02/30 Write block 209 len 4096 data: 7
69512024-02-27T21:46:42.350Z03/30 Read block 447 len 36864
69522024-02-27T21:46:42.350Z04/30 Read block 214 len 28672
69532024-02-27T21:46:42.350Z05/30 Read block 253 len 20480
69542024-02-27T21:46:42.357Z06/30 Write block 291 len 12288 data: 6 7 6
69552024-02-27T21:46:42.357Z07/30 Write block 4 len 32768 data: 3 3 4 4 4 7 8 8
69562024-02-27T21:46:42.357Z08/30 Read block 475 len 8192
69572024-02-27T21:46:42.357Z09/30 Write block 556 len 28672 data: 6 10 8 6 6 6 6
69582024-02-27T21:46:42.357Z10/30 Write block 275 len 16384 data: 3 5 5 5
69592024-02-27T21:46:42.357Z11/30 Read block 333 len 20480
69602024-02-27T21:46:42.357Z12/30 Write block 290 len 36864 data: 7 7 8 7 6 6 4 5 5
69612024-02-27T21:46:42.357Z13/30 Read block 170 len 36864
69622024-02-27T21:46:42.357Z14/30 Flush
69632024-02-27T21:46:42.357Z15/30 Write block 468 len 36864 data: 6 5 3 3 2 3 3 4 4
69642024-02-27T21:46:42.357Z16/30 Write block 489 len 4096 data: 9
69652024-02-27T21:46:42.357Z17/30 Read block 349 len 40960
69662024-02-27T21:46:42.357Z18/30 Read block 72 len 8192
69672024-02-27T21:46:42.357Z19/30 Flush
69682024-02-27T21:46:42.357Z20/30 Write block 283 len 8192 data: 5 3
69692024-02-27T21:46:42.357Z21/30 Write block 305 len 32768 data: 8 9 11 11 11 11 11 10
69702024-02-27T21:46:42.358Z22/30 Write block 293 len 36864 data: 8 7 7 5 6 6 4 5 5
69712024-02-27T21:46:42.358Z23/30 Write block 26 len 8192 data: 7 6
69722024-02-27T21:46:42.358Z24/30 Read block 528 len 40960
69732024-02-27T21:46:42.361Z25/30 Flush
69742024-02-27T21:46:42.362Z26/30 Write block 355 len 4096 data: 6
69752024-02-27T21:46:42.362Z27/30 Write block 212 len 24576 data: 7 5 5 4 6 9
69762024-02-27T21:46:42.362Z28/30 Read block 257 len 24576
69772024-02-27T21:46:42.362Z29/30 Write block 284 len 36864 data: 4 3 5 5 5 8 8 8 9
69782024-02-27T21:46:42.362Z30/30 Read block 132 len 16384
69792024-02-27T21:46:42.366Z----------------------------------------------------------------
69802024-02-27T21:46:42.366Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
69812024-02-27T21:46:42.367ZStates: Active Active Active
69822024-02-27T21:46:42.367ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69832024-02-27T21:46:42.367Z 1 Acked 1000 Write 8 Done Done Done false
69842024-02-27T21:46:42.367Z 2 Acked 1001 Write 1 Done Done Done false
69852024-02-27T21:46:42.367Z 3 Acked 1002 Read 9 Done Done Done false
69862024-02-27T21:46:42.367Z 4 Acked 1003 Read 7 Done Done Done false
69872024-02-27T21:46:42.367Z 5 Acked 1004 Read 5 Done Done Done false
69882024-02-27T21:46:42.368Z 6 Acked 1005 Write 3 Done Done Done false
69892024-02-27T21:46:42.368Z 7 Acked 1006 Write 8 Done Done Done false
69902024-02-27T21:46:42.368Z 8 Acked 1007 Read 2 Done Done Done false
69912024-02-27T21:46:42.368Z 9 Acked 1008 Write 7 Done Done Sent false
69922024-02-27T21:46:42.368Z 10 Acked 1009 Write 4 Done Done Sent false
69932024-02-27T21:46:42.368Z 11 Acked 1010 Read 5 Done Done Sent false
69942024-02-27T21:46:42.368Z 12 Acked 1011 Write 9 Done Done Sent false
69952024-02-27T21:46:42.368Z 13 Acked 1012 Read 9 Done Done Sent false
69962024-02-27T21:46:42.369Z 14 Acked 1013 Flush 0 Done Done Sent false
69972024-02-27T21:46:42.369Z 15 Acked 1014 Write 9 Done Done Sent false
69982024-02-27T21:46:42.369Z 16 Acked 1015 Write 1 Done Done Sent false
69992024-02-27T21:46:42.369Z 17 Acked 1016 Read 10 Done Done Sent false
70002024-02-27T21:46:42.369Z 18 Acked 1017 Read 2 Done Done Sent false
70012024-02-27T21:46:42.369Z 19 Acked 1018 Flush 0 Done Done Sent false
70022024-02-27T21:46:42.369Z 20 Acked 1019 Write 2 Done Done Sent false
70032024-02-27T21:46:42.369Z 21 Acked 1020 Write 8 Done Done Sent false
70042024-02-27T21:46:42.370Z 22 Acked 1021 Write 9 Done Done Sent false
70052024-02-27T21:46:42.370Z 23 Acked 1022 Write 2 Done Done Sent false
70062024-02-27T21:46:42.370Z 24 Acked 1023 Read 10 Done Done Sent false
70072024-02-27T21:46:42.370Z 25 Acked 1024 Flush 0 Done Done Sent false
70082024-02-27T21:46:42.370Z 26 Acked 1025 Write 1 Done Done Sent false
70092024-02-27T21:46:42.370Z 27 Acked 1026 Write 6 Done Done Sent false
70102024-02-27T21:46:42.371Z 28 Acked 1027 Read 6 Done Done Sent false
70112024-02-27T21:46:42.371Z 29 Acked 1028 Write 9 Done Done Sent false
70122024-02-27T21:46:42.371Z 30 Acked 1029 Read 4 Done Sent Sent false
70132024-02-27T21:46:42.371Z STATES DS:0 DS:1 DS:2 TOTAL
70142024-02-27T21:46:42.371Z New 0 0 0 0
70152024-02-27T21:46:42.371Z Sent 0 1 22 23
70162024-02-27T21:46:42.371Z Done 30 29 8 67
70172024-02-27T21:46:42.372Z Skipped 0 0 0 0
70182024-02-27T21:46:42.372Z Error 0 0 0 0
70192024-02-27T21:46:42.372ZLast Flush: 1024 1024 0
70202024-02-27T21:46:42.372ZDownstairs last five completed:
70212024-02-27T21:46:42.372ZUpstairs last five completed: 30 29 28 27 26
70222024-02-27T21:46:42.372Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:40.75952399Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1397}
70232024-02-27T21:46:42.372ZWrote out file "/tmp/test_repair_verify.data"
70242024-02-27T21:46:42.376Z
70252024-02-27T21:46:42.376ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70262024-02-27T21:46:42.387ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70272024-02-27T21:46:42.387Z 0 000-019 64 64 55 122 122 110 F F T <---
70282024-02-27T21:46:42.394Z 1 020-039 64 64 62 124 124 120 F F F <---
70292024-02-27T21:46:42.394Z 2 040-059 62 62 62 120 120 120 F F F
70302024-02-27T21:46:42.394Z 3 060-079 61 61 61 117 117 117 F F F
70312024-02-27T21:46:42.394Z 4 080-099 63 63 63 121 121 121 F F F
70322024-02-27T21:46:42.394Z 5 100-119 63 63 63 121 121 121 F F F
70332024-02-27T21:46:42.394Z 6 120-139 63 63 63 121 121 121 F F F
70342024-02-27T21:46:42.394Z 7 140-159 58 58 58 113 113 113 F F F
70352024-02-27T21:46:42.395Z 8 160-179 58 58 58 113 113 113 F F F
70362024-02-27T21:46:42.395Z 9 180-199 64 64 62 122 122 119 F F T <---
70372024-02-27T21:46:42.395Z 10 200-219 64 64 59 122 122 114 T T T <---
70382024-02-27T21:46:42.395Z 11 220-239 58 58 58 113 113 113 F F F
70392024-02-27T21:46:42.395Z 12 240-259 63 63 63 121 121 121 F F F
70402024-02-27T21:46:42.395Z 13 260-279 64 64 63 122 122 121 F F F <---
70412024-02-27T21:46:42.395Z 14 280-299 64 64 56 124 124 111 T T T <---
70422024-02-27T21:46:42.396Z 15 300-319 64 64 58 124 124 113 F F F <---
70432024-02-27T21:46:42.396Z 16 320-339 58 58 58 113 113 113 F F F
70442024-02-27T21:46:42.396Z 17 340-359 62 62 62 120 120 120 T T F <---
70452024-02-27T21:46:42.396Z 18 360-379 58 58 58 113 113 113 F F F
70462024-02-27T21:46:42.396Z 19 380-399 54 54 54 109 109 109 F F F
70472024-02-27T21:46:42.396Z 20 400-419 58 58 58 112 112 112 F F F
70482024-02-27T21:46:42.396Z 21 420-439 61 61 61 117 117 117 F F F
70492024-02-27T21:46:42.396Z 22 440-459 63 63 63 121 121 121 F F F
70502024-02-27T21:46:42.397Z 23 460-479 64 64 62 123 123 120 F F F <---
70512024-02-27T21:46:42.397Z 24 480-499 64 64 50 123 123 101 F F F <---
70522024-02-27T21:46:42.397Z 25 500-519 62 62 62 120 120 120 F F F
70532024-02-27T21:46:42.397Z 26 520-539 56 56 56 111 111 111 F F F
70542024-02-27T21:46:42.397Z 27 540-559 64 64 61 122 122 117 F F F <---
70552024-02-27T21:46:42.397Z 28 560-579 64 64 58 122 122 113 F F F <---
70562024-02-27T21:46:42.397Z 29 580-599 61 61 61 117 117 117 F F F
70572024-02-27T21:46:42.397ZMax gen: 64, Max flush: 124
70582024-02-27T21:46:42.397ZError: Difference in extent metadata found!
70592024-02-27T21:46:42.397ZA Difference in extent metadata is expected here
70602024-02-27T21:46:42.398ZOn loop 32
70612024-02-27T21:46:42.398Z
70622024-02-27T21:46:42.403ZVerifying data now
70632024-02-27T21:46:48.000ZNo change after verify
70642024-02-27T21:46:48.000ZLoop: 32 Downstairs dump after verify (and repair):
70652024-02-27T21:46:48.021ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70662024-02-27T21:46:48.021Z 0 000-019 64 64 64 122 122 122 F F F
70672024-02-27T21:46:48.021Z 1 020-039 64 64 64 124 124 124 F F F
70682024-02-27T21:46:48.021Z 2 040-059 62 62 62 120 120 120 F F F
70692024-02-27T21:46:48.022Z 3 060-079 61 61 61 117 117 117 F F F
70702024-02-27T21:46:48.022Z 4 080-099 63 63 63 121 121 121 F F F
70712024-02-27T21:46:48.022Z 5 100-119 63 63 63 121 121 121 F F F
70722024-02-27T21:46:48.022Z 6 120-139 63 63 63 121 121 121 F F F
70732024-02-27T21:46:48.022Z 7 140-159 58 58 58 113 113 113 F F F
70742024-02-27T21:46:48.022Z 8 160-179 58 58 58 113 113 113 F F F
70752024-02-27T21:46:48.023Z 9 180-199 64 64 64 122 122 122 F F F
70762024-02-27T21:46:48.023Z 10 200-219 65 65 65 125 125 125 F F F
70772024-02-27T21:46:48.023Z 11 220-239 58 58 58 113 113 113 F F F
70782024-02-27T21:46:48.023Z 12 240-259 63 63 63 121 121 121 F F F
70792024-02-27T21:46:48.023Z 13 260-279 64 64 64 122 122 122 F F F
70802024-02-27T21:46:48.023Z 14 280-299 65 65 65 125 125 125 F F F
70812024-02-27T21:46:48.023Z 15 300-319 64 64 64 124 124 124 F F F
70822024-02-27T21:46:48.023Z 16 320-339 58 58 58 113 113 113 F F F
70832024-02-27T21:46:48.024Z 17 340-359 65 65 65 125 125 125 F F F
70842024-02-27T21:46:48.024Z 18 360-379 58 58 58 113 113 113 F F F
70852024-02-27T21:46:48.024Z 19 380-399 54 54 54 109 109 109 F F F
70862024-02-27T21:46:48.025Z 20 400-419 58 58 58 112 112 112 F F F
70872024-02-27T21:46:48.025Z 21 420-439 61 61 61 117 117 117 F F F
70882024-02-27T21:46:48.025Z 22 440-459 63 63 63 121 121 121 F F F
70892024-02-27T21:46:48.082Z 23 460-479 64 64 64 123 123 123 F F F
70902024-02-27T21:46:48.082Z 24 480-499 64 64 64 123 123 123 F F F
70912024-02-27T21:46:48.082Z 25 500-519 62 62 62 120 120 120 F F F
70922024-02-27T21:46:48.082Z 26 520-539 56 56 56 111 111 111 F F F
70932024-02-27T21:46:48.082Z 27 540-559 64 64 64 122 122 122 F F F
70942024-02-27T21:46:48.082Z 28 560-579 64 64 64 122 122 122 F F F
70952024-02-27T21:46:48.082Z 29 580-599 61 61 61 117 117 117 F F F
70962024-02-27T21:46:48.083ZMax gen: 65, Max flush: 125
70972024-02-27T21:46:48.083Z
70982024-02-27T21:46:48.083ZBegin loop 33
70992024-02-27T21:46:48.083ZDownstairs to restart: 0
71002024-02-27T21:46:48.083Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.42992769Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405}
71012024-02-27T21:46:48.083Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430198388Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405}
71022024-02-27T21:46:48.083Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430221145Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405}
71032024-02-27T21:46:48.083Z{"msg":"Crucible 0fbbc811-e5c6-4da9-a6ac-2298b78de6df has session id: 42ec74c5-65e3-417d-96e7-ae3fef9f4f4f","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430253991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71042024-02-27T21:46:48.083Z{"msg":"Crucible runtime is spawned
71052024-02-27T21:46:48.084ZUpstairs opts: Upstairs UUID: 0fbbc811-e5c6-4da9-a6ac-2298b78de6df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430285658Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71062024-02-27T21:46:48.084Z{"msg":"Crucible stats registered with UUID: 0fbbc811-e5c6-4da9-a6ac-2298b78de6df","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430312604Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71072024-02-27T21:46:48.084Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430345211Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405}
71082024-02-27T21:46:48.084Z{"msg":"0fbbc811-e5c6-4da9-a6ac-2298b78de6df active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430393625Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71092024-02-27T21:46:48.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430425181Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71102024-02-27T21:46:48.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430448968Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71112024-02-27T21:46:48.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430485624Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71122024-02-27T21:46:48.085Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430507721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71132024-02-27T21:46:48.085Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430528149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71142024-02-27T21:46:48.085Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430559775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71152024-02-27T21:46:48.085Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430575613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71162024-02-27T21:46:48.085Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430600161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71172024-02-27T21:46:48.085Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430641336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71182024-02-27T21:46:48.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430818515Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71192024-02-27T21:46:48.086Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430919633Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71202024-02-27T21:46:48.086Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.430965138Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71212024-02-27T21:46:48.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.431675644Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71222024-02-27T21:46:48.086Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.431713319Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71232024-02-27T21:46:48.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.431729577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71242024-02-27T21:46:48.086Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.431822426Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71252024-02-27T21:46:48.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.431868211Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71262024-02-27T21:46:48.086Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.431912386Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71272024-02-27T21:46:48.087Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.432335256Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71282024-02-27T21:46:48.087Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.432382121Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71292024-02-27T21:46:48.087Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.432411307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71302024-02-27T21:46:48.087Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.432969771Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71312024-02-27T21:46:48.087Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433008437Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71322024-02-27T21:46:48.087Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433025275Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71332024-02-27T21:46:48.087Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433038823Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71342024-02-27T21:46:48.087Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433060511Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71352024-02-27T21:46:48.088Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433074239Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71362024-02-27T21:46:48.088Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433093607Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71372024-02-27T21:46:48.088Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433107535Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71382024-02-27T21:46:48.088Z{"msg":"[0]R flush_numbers[0..12]: [122, 124, 120, 117, 121, 121, 121, 113, 113, 122, 125, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433129683Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71392024-02-27T21:46:48.088Z{"msg":"[0]R generation[0..12]: [64, 64, 62, 61, 63, 63, 63, 58, 58, 64, 65, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433144351Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71402024-02-27T21:46:48.088ZWait for a query_work_queue command to finish before sending IO
71412024-02-27T21:46:48.089Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433170368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71422024-02-27T21:46:48.089Z{"msg":"[1]R flush_numbers[0..12]: [122, 124, 120, 117, 121, 121, 121, 113, 113, 122, 125, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433184766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71432024-02-27T21:46:48.089Z{"msg":"[1]R generation[0..12]: [64, 64, 62, 61, 63, 63, 63, 58, 58, 64, 65, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433205194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71442024-02-27T21:46:48.089Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433220452Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71452024-02-27T21:46:48.089Z{"msg":"[2]R flush_numbers[0..12]: [122, 124, 120, 117, 121, 121, 121, 113, 113, 122, 125, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.43324149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71462024-02-27T21:46:48.089Z{"msg":"[2]R generation[0..12]: [64, 64, 62, 61, 63, 63, 63, 58, 58, 64, 65, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433255888Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71472024-02-27T21:46:48.089Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433270826Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71482024-02-27T21:46:48.089Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433284364Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71492024-02-27T21:46:48.089Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433298443Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71502024-02-27T21:46:48.090Z{"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433311421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71512024-02-27T21:46:48.090Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.43332514Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"":"downstairs","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71522024-02-27T21:46:48.090Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433338508Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71532024-02-27T21:46:48.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433351106Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"0","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71542024-02-27T21:46:48.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433364655Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"1","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71552024-02-27T21:46:48.090Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433384613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"client":"2","":"downstairs","session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71562024-02-27T21:46:48.090Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433398261Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71572024-02-27T21:46:48.090Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433417659Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71582024-02-27T21:46:48.090Z{"msg":"0fbbc811-e5c6-4da9-a6ac-2298b78de6df is now active with session: 42ec74c5-65e3-417d-96e7-ae3fef9f4f4f","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433431037Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71592024-02-27T21:46:48.091Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433443096Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405,"session_id":"42ec74c5-65e3-417d-96e7-ae3fef9f4f4f"}
71602024-02-27T21:46:48.091Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:46.433461773Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405}
71612024-02-27T21:46:48.091ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71622024-02-27T21:46:48.091ZLoading write count information from file "/tmp/test_repair_verify.data"
71632024-02-27T21:46:48.091ZRun Repair workload
71642024-02-27T21:46:48.091Z01/30 Write block 589 len 32768 data: 4 3 3 4 4 4 4 5
71652024-02-27T21:46:48.091Z02/30 Read block 167 len 12288
71662024-02-27T21:46:48.091Z03/30 Read block 580 len 36864
71672024-02-27T21:46:48.091Z04/30 Read block 309 len 24576
71682024-02-27T21:46:48.091Z05/30 Read block 476 len 12288
71692024-02-27T21:46:48.092Z06/30 Read block 572 len 32768
71702024-02-27T21:46:48.092Z07/30 Read block 99 len 8192
71712024-02-27T21:46:48.092Z08/30 Read block 106 len 36864
71722024-02-27T21:46:48.092Z09/30 Read block 487 len 8192
71732024-02-27T21:46:48.092Z10/30 Write block 108 len 32768 data: 8 9 8 9 7 5 5 5
71742024-02-27T21:46:48.092Z11/30 Flush
71752024-02-27T21:46:48.092Z12/30 Write block 404 len 40960 data: 4 3 4 3 4 5 4 3 5 5
71762024-02-27T21:46:48.092Z13/30 Read block 359 len 12288
71772024-02-27T21:46:48.093Z14/30 Read block 169 len 24576
71782024-02-27T21:46:48.093Z15/30 Read block 37 len 20480
71792024-02-27T21:46:48.093Z16/30 Read block 80 len 12288
71802024-02-27T21:46:48.093Z17/30 Write block 398 len 16384 data: 4 5 6 7
71812024-02-27T21:46:48.093Z18/30 Read block 91 len 8192
71822024-02-27T21:46:48.093Z19/30 Flush
71832024-02-27T21:46:48.093Z20/30 Write block 598 len 4096 data: 4
71842024-02-27T21:46:48.094Z21/30 Write block 163 len 12288 data: 5 4 3
71852024-02-27T21:46:48.094Z22/30 Read block 204 len 40960
71862024-02-27T21:46:48.094Z23/30 Read block 570 len 16384
71872024-02-27T21:46:48.094Z24/30 Write block 397 len 36864 data: 6 5 6 7 8 5 4 5 4
71882024-02-27T21:46:48.094Z25/30 Read block 373 len 40960
71892024-02-27T21:46:48.145Z26/30 Write block 226 len 12288 data: 7 7 9
71902024-02-27T21:46:48.145Z27/30 Write block 155 len 12288 data: 6 8 8
71912024-02-27T21:46:48.145Z28/30 Write block 48 len 28672 data: 3 4 5 4 5 6 5
71922024-02-27T21:46:48.145Z29/30 Write block 388 len 16384 data: 8 9 7 7
71932024-02-27T21:46:48.146Z30/30 Read block 74 len 40960
71942024-02-27T21:46:48.146Z----------------------------------------------------------------
71952024-02-27T21:46:48.146Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
71962024-02-27T21:46:48.146ZStates: Active Active Active
71972024-02-27T21:46:48.146ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
71982024-02-27T21:46:48.146Z 1 Acked 1000 Write 8 Done Done Done false
71992024-02-27T21:46:48.146Z 2 Acked 1001 Read 3 Done Done Done false
72002024-02-27T21:46:48.146Z 3 Acked 1002 Read 9 Sent Done Done false
72012024-02-27T21:46:48.147Z 4 Acked 1003 Read 6 Sent Done Done false
72022024-02-27T21:46:48.147Z 5 Acked 1004 Read 3 Sent Done Done false
72032024-02-27T21:46:48.147Z 6 Acked 1005 Read 8 Sent Done Done false
72042024-02-27T21:46:48.147Z 7 Acked 1006 Read 2 Sent Done Done false
72052024-02-27T21:46:48.147Z 8 Acked 1007 Read 9 Sent Done Done false
72062024-02-27T21:46:48.147Z 9 Acked 1008 Read 2 Sent Done Done false
72072024-02-27T21:46:48.147Z 10 Acked 1009 Write 8 Sent Done Done false
72082024-02-27T21:46:48.148Z 11 Acked 1010 Flush 0 Sent Done Done false
72092024-02-27T21:46:48.148Z 12 Acked 1011 Write 10 Sent Done Done false
72102024-02-27T21:46:48.148Z 13 Acked 1012 Read 3 Sent Done Done false
72112024-02-27T21:46:48.148Z 14 Acked 1013 Read 6 Sent Done Done false
72122024-02-27T21:46:48.148Z 15 Acked 1014 Read 5 Sent Done Done false
72132024-02-27T21:46:48.148Z 16 Acked 1015 Read 3 Sent Done Done false
72142024-02-27T21:46:48.148Z 17 Acked 1016 Write 4 Sent Done Done false
72152024-02-27T21:46:48.148Z 18 Acked 1017 Read 2 Sent Done Done false
72162024-02-27T21:46:48.148Z 19 Acked 1018 Flush 0 Sent Done Done false
72172024-02-27T21:46:48.148Z 20 Acked 1019 Write 1 Sent Done Done false
72182024-02-27T21:46:48.149Z 21 Acked 1020 Write 3 Sent Done Done false
72192024-02-27T21:46:48.149Z 22 Acked 1021 Read 10 Sent Done Done false
72202024-02-27T21:46:48.149Z 23 Acked 1022 Read 4 Sent Done Done false
72212024-02-27T21:46:48.149Z 24 Acked 1023 Write 9 Sent Done Done false
72222024-02-27T21:46:48.149Z 25 Acked 1024 Read 10 Sent Done Done false
72232024-02-27T21:46:48.149Z 26 Acked 1025 Write 3 Sent Done Done false
72242024-02-27T21:46:48.149Z 27 Acked 1026 Write 3 Sent Done Done false
72252024-02-27T21:46:48.149Z 28 Acked 1027 Write 7 Sent Done Done false
72262024-02-27T21:46:48.149Z 29 Acked 1028 Write 4 Sent Done Done false
72272024-02-27T21:46:48.150Z 30 Acked 1029 Read 10 Sent Done Done false
72282024-02-27T21:46:48.150Z STATES DS:0 DS:1 DS:2 TOTAL
72292024-02-27T21:46:48.150Z New 0 0 0 0
72302024-02-27T21:46:48.150Z Sent 28 0 0 28
72312024-02-27T21:46:48.150Z Done 2 30 30 62
72322024-02-27T21:46:48.150Z Skipped 0 0 0 0
72332024-02-27T21:46:48.150Z Error 0 0 0 0
72342024-02-27T21:46:48.150ZLast Flush: 0 1018 1018
72352024-02-27T21:46:48.150ZDownstairs last five completed:
72362024-02-27T21:46:48.151ZUpstairs last five completed: 30 29 28 27 26
72372024-02-27T21:46:48.151Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:46.462961779Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1405}
72382024-02-27T21:46:48.151ZWrote out file "/tmp/test_repair_verify.data"
72392024-02-27T21:46:48.151Z
72402024-02-27T21:46:48.151ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72412024-02-27T21:46:48.151ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72422024-02-27T21:46:48.151Z 0 000-019 64 64 64 122 122 122 F F F
72432024-02-27T21:46:48.151Z 1 020-039 64 64 64 124 124 124 F F F
72442024-02-27T21:46:48.151Z 2 040-059 62 62 62 120 120 120 F T T <---
72452024-02-27T21:46:48.151Z 3 060-079 61 61 61 117 117 117 F F F
72462024-02-27T21:46:48.151Z 4 080-099 63 63 63 121 121 121 F F F
72472024-02-27T21:46:48.151Z 5 100-119 63 66 66 121 126 126 F F F <---
72482024-02-27T21:46:48.151Z 6 120-139 63 63 63 121 121 121 F F F
72492024-02-27T21:46:48.152Z 7 140-159 58 58 58 113 113 113 F T T <---
72502024-02-27T21:46:48.152Z 8 160-179 58 58 58 113 113 113 F T T <---
72512024-02-27T21:46:48.152Z 9 180-199 64 64 64 122 122 122 F F F
72522024-02-27T21:46:48.152Z 10 200-219 65 65 65 125 125 125 F F F
72532024-02-27T21:46:48.152Z 11 220-239 58 58 58 113 113 113 F T T <---
72542024-02-27T21:46:48.152Z 12 240-259 63 63 63 121 121 121 F F F
72552024-02-27T21:46:48.152Z 13 260-279 64 64 64 122 122 122 F F F
72562024-02-27T21:46:48.152Z 14 280-299 65 65 65 125 125 125 F F F
72572024-02-27T21:46:48.152Z 15 300-319 64 64 64 124 124 124 F F F
72582024-02-27T21:46:48.152Z 16 320-339 58 58 58 113 113 113 F F F
72592024-02-27T21:46:48.152Z 17 340-359 65 65 65 125 125 125 F F F
72602024-02-27T21:46:48.153Z 18 360-379 58 58 58 113 113 113 F F F
72612024-02-27T21:46:48.153Z 19 380-399 54 66 66 109 127 127 F T T <---
72622024-02-27T21:46:48.153Z 20 400-419 58 66 66 112 127 127 F T T <---
72632024-02-27T21:46:48.153Z 21 420-439 61 61 61 117 117 117 F F F
72642024-02-27T21:46:48.153Z 22 440-459 63 63 63 121 121 121 F F F
72652024-02-27T21:46:48.153Z 23 460-479 64 64 64 123 123 123 F F F
72662024-02-27T21:46:48.153Z 24 480-499 64 64 64 123 123 123 F F F
72672024-02-27T21:46:48.153Z 25 500-519 62 62 62 120 120 120 F F F
72682024-02-27T21:46:48.153Z 26 520-539 56 56 56 111 111 111 F F F
72692024-02-27T21:46:48.153Z 27 540-559 64 64 64 122 122 122 F F F
72702024-02-27T21:46:48.153Z 28 560-579 64 64 64 122 122 122 F F F
72712024-02-27T21:46:48.153Z 29 580-599 61 66 66 117 126 126 T T T <---
72722024-02-27T21:46:48.154ZMax gen: 66, Max flush: 127
72732024-02-27T21:46:48.154ZError: Difference in extent metadata found!
72742024-02-27T21:46:48.154ZA Difference in extent metadata is expected here
72752024-02-27T21:46:48.154ZOn loop 33
72762024-02-27T21:46:48.154Z
72772024-02-27T21:46:48.154ZVerifying data now
72782024-02-27T21:46:52.547ZNo change after verify
72792024-02-27T21:46:52.547ZLoop: 33 Downstairs dump after verify (and repair):
72802024-02-27T21:46:52.554ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72812024-02-27T21:46:52.555Z 0 000-019 64 64 64 122 122 122 F F F
72822024-02-27T21:46:52.555Z 1 020-039 64 64 64 124 124 124 F F F
72832024-02-27T21:46:52.555Z 2 040-059 67 67 67 128 128 128 F F F
72842024-02-27T21:46:52.555Z 3 060-079 61 61 61 117 117 117 F F F
72852024-02-27T21:46:52.556Z 4 080-099 63 63 63 121 121 121 F F F
72862024-02-27T21:46:52.561Z 5 100-119 66 66 66 126 126 126 F F F
72872024-02-27T21:46:52.561Z 6 120-139 63 63 63 121 121 121 F F F
72882024-02-27T21:46:52.561Z 7 140-159 67 67 67 128 128 128 F F F
72892024-02-27T21:46:52.562Z 8 160-179 67 67 67 128 128 128 F F F
72902024-02-27T21:46:52.562Z 9 180-199 64 64 64 122 122 122 F F F
72912024-02-27T21:46:52.562Z 10 200-219 65 65 65 125 125 125 F F F
72922024-02-27T21:46:52.562Z 11 220-239 67 67 67 128 128 128 F F F
72932024-02-27T21:46:52.562Z 12 240-259 63 63 63 121 121 121 F F F
72942024-02-27T21:46:52.562Z 13 260-279 64 64 64 122 122 122 F F F
72952024-02-27T21:46:52.562Z 14 280-299 65 65 65 125 125 125 F F F
72962024-02-27T21:46:52.562Z 15 300-319 64 64 64 124 124 124 F F F
72972024-02-27T21:46:52.562Z 16 320-339 58 58 58 113 113 113 F F F
72982024-02-27T21:46:52.563Z 17 340-359 65 65 65 125 125 125 F F F
72992024-02-27T21:46:52.563Z 18 360-379 58 58 58 113 113 113 F F F
73002024-02-27T21:46:52.563Z 19 380-399 67 67 67 128 128 128 F F F
73012024-02-27T21:46:52.563Z 20 400-419 67 67 67 128 128 128 F F F
73022024-02-27T21:46:52.563Z 21 420-439 61 61 61 117 117 117 F F F
73032024-02-27T21:46:52.563Z 22 440-459 63 63 63 121 121 121 F F F
73042024-02-27T21:46:52.563Z 23 460-479 64 64 64 123 123 123 F F F
73052024-02-27T21:46:52.563Z 24 480-499 64 64 64 123 123 123 F F F
73062024-02-27T21:46:52.563Z 25 500-519 62 62 62 120 120 120 F F F
73072024-02-27T21:46:52.564Z 26 520-539 56 56 56 111 111 111 F F F
73082024-02-27T21:46:52.564Z 27 540-559 64 64 64 122 122 122 F F F
73092024-02-27T21:46:52.564Z 28 560-579 64 64 64 122 122 122 F F F
73102024-02-27T21:46:52.564Z 29 580-599 67 67 67 128 128 128 F F F
73112024-02-27T21:46:52.564ZMax gen: 67, Max flush: 128
73122024-02-27T21:46:52.564Z
73132024-02-27T21:46:52.564ZBegin loop 34
73142024-02-27T21:46:52.564ZDownstairs to restart: 2
73152024-02-27T21:46:52.577Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972117885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413}
73162024-02-27T21:46:52.577Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972419529Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413Crucible runtime is spawned
73172024-02-27T21:46:52.578Z}
73182024-02-27T21:46:52.638Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972471633Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413}
73192024-02-27T21:46:52.638Z{"msg":"Crucible 52c2abc6-ca1e-4736-be20-ae2844ecec24 has session id: b61790b4-c37d-4706-9034-af8cbb612fbb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972512758Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73202024-02-27T21:46:52.638Z{"msg":"Upstairs opts: Upstairs UUID: 52c2abc6-ca1e-4736-be20-ae2844ecec24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972570571Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73212024-02-27T21:46:52.638Z{"msg":"Crucible stats registered with UUID: 52c2abc6-ca1e-4736-be20-ae2844ecec24","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972619175Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73222024-02-27T21:46:52.638Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972649972Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413}
73232024-02-27T21:46:52.638Z{"msg":"52c2abc6-ca1e-4736-be20-ae2844ecec24 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.97266588Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73242024-02-27T21:46:52.638Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972692027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73252024-02-27T21:46:52.638Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972715084Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73262024-02-27T21:46:52.639Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972732872Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73272024-02-27T21:46:52.639Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972760119Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73282024-02-27T21:46:52.639Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972789555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73292024-02-27T21:46:52.639Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972816442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73302024-02-27T21:46:52.639Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972840449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73312024-02-27T21:46:52.639Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972880015Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73322024-02-27T21:46:52.640Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972926899Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73332024-02-27T21:46:52.640Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.972963695Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73342024-02-27T21:46:52.640Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.973027727Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73352024-02-27T21:46:52.640Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.973074502Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73362024-02-27T21:46:52.640Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.973750152Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73372024-02-27T21:46:52.641Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.973798986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73382024-02-27T21:46:52.641Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.973830683Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73392024-02-27T21:46:52.642Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.973884156Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73402024-02-27T21:46:52.642Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.974455379Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73412024-02-27T21:46:52.642Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.974605351Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73422024-02-27T21:46:52.642Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.974816376Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73432024-02-27T21:46:52.642Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.97487023Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73442024-02-27T21:46:52.642Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.974932373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73452024-02-27T21:46:52.642Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.974977158Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73462024-02-27T21:46:52.642Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.975006694Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73472024-02-27T21:46:52.643Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.975285201Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73482024-02-27T21:46:52.643Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.975337955Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73492024-02-27T21:46:52.643Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.97538392Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73502024-02-27T21:46:52.643Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.975575277Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73512024-02-27T21:46:52.643Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976026964Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73522024-02-27T21:46:52.643Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.9760646Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73532024-02-27T21:46:52.643Z{"msg":"[0]R flush_numbers[0..12]: [122, 124, 128, 117, 121, 126, 121, 128, 128, 122, 125, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976082907Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73542024-02-27T21:46:52.644Z{"msg":"[0]R generation[0..12]: [64, 64, 67, 61, 63, 66, 63, 67, 67, 64, 65, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976097716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73552024-02-27T21:46:52.644Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976119773Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73562024-02-27T21:46:52.644Z{"msg":"[1]R flush_numbers[0..12]: [122, 124, 128, 117, 121, 126, 121, 128, 128, 122, 125, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.97614714Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73572024-02-27T21:46:52.644Z{"msg":"[1]R generation[0..12]: [64, 64, 67, 61, 63, 66, 63, 67, 67, 64, 65, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976176266Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73582024-02-27T21:46:52.644Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976202543Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73592024-02-27T21:46:52.645Z{"msg":"Wait for a query_work_queue command to finish before sending IO
73602024-02-27T21:46:52.645Z[2]R flush_numbers[0..12]: [122, 124, 128, 117, 121, 126, 121, 128, 128, 122, 125, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.9762322Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73612024-02-27T21:46:52.645Z{"msg":"[2]R generation[0..12]: [64, 64, 67, 61, 63, 66, 63, 67, 67, 64, 65, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976258367Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73622024-02-27T21:46:52.645Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976292513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73632024-02-27T21:46:52.645Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976324869Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73642024-02-27T21:46:52.645Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976347296Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73652024-02-27T21:46:52.645Z{"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976378033Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73662024-02-27T21:46:52.646Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.97640232Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"":"downstairs","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73672024-02-27T21:46:52.646Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976429506Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73682024-02-27T21:46:52.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976451844Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"0","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73692024-02-27T21:46:52.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976478981Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"1","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73702024-02-27T21:46:52.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976502648Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"client":"2","":"downstairs","session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73712024-02-27T21:46:52.653Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976529625Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73722024-02-27T21:46:52.654Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976551862Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73732024-02-27T21:46:52.654Z{"msg":"52c2abc6-ca1e-4736-be20-ae2844ecec24 is now active with session: b61790b4-c37d-4706-9034-af8cbb612fbb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976579649Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73742024-02-27T21:46:52.654Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976602246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413,"session_id":"b61790b4-c37d-4706-9034-af8cbb612fbb"}
73752024-02-27T21:46:52.654ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73762024-02-27T21:46:52.654Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:50.976639242Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413}
73772024-02-27T21:46:52.654ZLoading write count information from file "/tmp/test_repair_verify.data"
73782024-02-27T21:46:52.654ZRun Repair workload
73792024-02-27T21:46:52.655Z01/30 Write block 124 len 4096 data: 7
73802024-02-27T21:46:52.655Z02/30 Flush
73812024-02-27T21:46:52.673Z03/30 Write block 111 len 32768 data: 10 8 6 6 6 5 7 7
73822024-02-27T21:46:52.674Z04/30 Write block 334 len 8192 data: 7 8
73832024-02-27T21:46:52.674Z05/30 Read block 2 len 40960
73842024-02-27T21:46:52.674Z06/30 Read block 228 len 8192
73852024-02-27T21:46:52.674Z07/30 Write block 528 len 4096 data: 4
73862024-02-27T21:46:52.674Z08/30 Read block 372 len 16384
73872024-02-27T21:46:52.674Z09/30 Read block 472 len 28672
73882024-02-27T21:46:52.674Z10/30 Write block 506 len 28672 data: 6 6 4 7 9 13 12
73892024-02-27T21:46:52.675Z11/30 Write block 124 len 8192 data: 8 6
73902024-02-27T21:46:52.675Z12/30 Flush
73912024-02-27T21:46:52.675Z13/30 Write block 209 len 40960 data: 8 7 7 8 6 6 5 7 10 10
73922024-02-27T21:46:52.675Z14/30 Write block 483 len 36864 data: 8 9 9 9 8 8 10 8 8
73932024-02-27T21:46:52.675Z15/30 Read block 148 len 16384
73942024-02-27T21:46:52.675Z16/30 Read block 514 len 12288
73952024-02-27T21:46:52.675Z17/30 Read block 206 len 36864
73962024-02-27T21:46:52.675Z18/30 Write block 91 len 4096 data: 6
73972024-02-27T21:46:52.675Z19/30 Write block 293 len 28672 data: 9 8 8 6 7 7 5
73982024-02-27T21:46:52.675Z20/30 Read block 516 len 12288
73992024-02-27T21:46:52.675Z21/30 Write block 180 len 24576 data: 7 9 9 9 8 5
74002024-02-27T21:46:52.675Z22/30 Read block 180 len 36864
74012024-02-27T21:46:52.675Z23/30 Read block 54 len 8192
74022024-02-27T21:46:52.675Z24/30 Write block 115 len 40960 data: 7 6 8 8 7 8 8 8 8 9
74032024-02-27T21:46:52.675Z25/30 Flush
74042024-02-27T21:46:52.675Z26/30 Flush
74052024-02-27T21:46:52.676Z27/30 Write block 431 len 20480 data: 5 6 8 10 7
74062024-02-27T21:46:52.676Z28/30 Write block 42 len 28672 data: 6 7 6 5 3 3 4
74072024-02-27T21:46:52.676Z29/30 Write block 590 len 16384 data: 4 4 5 5
74082024-02-27T21:46:52.676Z30/30 Write block 561 len 16384 data: 7 7 5 2
74092024-02-27T21:46:52.676Z----------------------------------------------------------------
74102024-02-27T21:46:52.676Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:28
74112024-02-27T21:46:52.676ZStates: Active Active Active
74122024-02-27T21:46:52.676ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74132024-02-27T21:46:52.676Z 3 Acked 1002 Write 8 Done Done Sent false
74142024-02-27T21:46:52.676Z 4 Acked 1003 Write 2 Done Done Sent false
74152024-02-27T21:46:52.676Z 5 Acked 1004 Read 10 Done Done Sent false
74162024-02-27T21:46:52.676Z 6 Acked 1005 Read 2 Done Done Sent false
74172024-02-27T21:46:52.676Z 7 Acked 1006 Write 1 Done Done Sent false
74182024-02-27T21:46:52.676Z 8 Acked 1007 Read 4 Done Done Sent false
74192024-02-27T21:46:52.676Z 9 Acked 1008 Read 7 Done Done Sent false
74202024-02-27T21:46:52.676Z 10 Acked 1009 Write 7 Done Done Sent false
74212024-02-27T21:46:52.676Z 11 Acked 1010 Write 2 Done Done Sent false
74222024-02-27T21:46:52.676Z 12 Acked 1011 Flush 0 Done Done Sent false
74232024-02-27T21:46:52.677Z 13 Acked 1012 Write 10 Done Done Sent false
74242024-02-27T21:46:52.677Z 14 Acked 1013 Write 9 Done Done Sent false
74252024-02-27T21:46:52.677Z 15 Acked 1014 Read 4 Done Done Sent false
74262024-02-27T21:46:52.677Z 16 Acked 1015 Read 3 Done Done Sent false
74272024-02-27T21:46:52.677Z 17 Acked 1016 Read 9 Done Done Sent false
74282024-02-27T21:46:52.677Z 18 Acked 1017 Write 1 Done Done Sent false
74292024-02-27T21:46:52.677Z 19 Acked 1018 Write 7 Done Done Sent false
74302024-02-27T21:46:52.677Z 20 Acked 1019 Read 3 Done Done Sent false
74312024-02-27T21:46:52.677Z 21 Acked 1020 Write 6 Done Done Sent false
74322024-02-27T21:46:52.677Z 22 Acked 1021 Read 9 Done Done Sent false
74332024-02-27T21:46:52.677Z 23 Acked 1022 Read 2 Done Done Sent false
74342024-02-27T21:46:52.677Z 24 Acked 1023 Write 10 Done Done Sent false
74352024-02-27T21:46:52.677Z 25 Acked 1024 Flush 0 Done Done Sent false
74362024-02-27T21:46:52.677Z 26 Acked 1025 Flush 0 Done Done Sent false
74372024-02-27T21:46:52.677Z 27 Acked 1026 Write 5 Done Done Sent false
74382024-02-27T21:46:52.677Z 28 Acked 1027 Write 7 Done Done Sent false
74392024-02-27T21:46:52.677Z 29 Acked 1028 Write 4 Done Done Sent false
74402024-02-27T21:46:52.677Z 30 Acked 1029 Write 4 Sent Sent Sent false
74412024-02-27T21:46:52.677Z STATES DS:0 DS:1 DS:2 TOTAL
74422024-02-27T21:46:52.677Z New 0 0 0 0
74432024-02-27T21:46:52.677Z Sent 1 1 28 30
74442024-02-27T21:46:52.677Z Done 27 27 0 54
74452024-02-27T21:46:52.678Z Skipped 0 0 0 0
74462024-02-27T21:46:52.678Z Error 0 0 0 0
74472024-02-27T21:46:52.678ZLast Flush: 1025 1025 1001
74482024-02-27T21:46:52.678ZDownstairs last five completed: 1001 1000
74492024-02-27T21:46:52.678ZUpstairs last five completed: 30 29 28 27 26
74502024-02-27T21:46:52.678Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:51.007741648Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1413}
74512024-02-27T21:46:52.678ZWrote out file "/tmp/test_repair_verify.data"
74522024-02-27T21:46:52.678Z
74532024-02-27T21:46:52.678ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74542024-02-27T21:46:52.678ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74552024-02-27T21:46:52.678Z 0 000-019 64 64 64 122 122 122 F F F
74562024-02-27T21:46:52.678Z 1 020-039 64 64 64 124 124 124 F F F
74572024-02-27T21:46:52.678Z 2 040-059 67 67 67 128 128 128 T T F <---
74582024-02-27T21:46:52.678Z 3 060-079 61 61 61 117 117 117 F F F
74592024-02-27T21:46:52.678Z 4 080-099 68 68 63 131 131 121 F F F <---
74602024-02-27T21:46:52.678Z 5 100-119 68 68 66 131 131 126 F F F <---
74612024-02-27T21:46:52.678Z 6 120-139 68 68 68 131 131 129 F F F <---
74622024-02-27T21:46:52.678Z 7 140-159 67 67 67 128 128 128 F F F
74632024-02-27T21:46:52.678Z 8 160-179 67 67 67 128 128 128 F F F
74642024-02-27T21:46:52.678Z 9 180-199 68 68 64 131 131 122 F F F <---
74652024-02-27T21:46:52.679Z 10 200-219 68 68 65 131 131 125 F F F <---
74662024-02-27T21:46:52.679Z 11 220-239 67 67 67 128 128 128 F F F
74672024-02-27T21:46:52.679Z 12 240-259 63 63 63 121 121 121 F F F
74682024-02-27T21:46:52.679Z 13 260-279 64 64 64 122 122 122 F F F
74692024-02-27T21:46:52.679Z 14 280-299 68 68 65 131 131 125 F F F <---
74702024-02-27T21:46:52.679Z 15 300-319 64 64 64 124 124 124 F F F
74712024-02-27T21:46:52.679Z 16 320-339 68 68 58 130 130 113 F F F <---
74722024-02-27T21:46:52.679Z 17 340-359 65 65 65 125 125 125 F F F
74732024-02-27T21:46:52.679Z 18 360-379 58 58 58 113 113 113 F F F
74742024-02-27T21:46:52.679Z 19 380-399 67 67 67 128 128 128 F F F
74752024-02-27T21:46:52.679Z 20 400-419 67 67 67 128 128 128 F F F
74762024-02-27T21:46:52.679Z 21 420-439 61 61 61 117 117 117 T T F <---
74772024-02-27T21:46:52.679Z 22 440-459 63 63 63 121 121 121 F F F
74782024-02-27T21:46:52.679Z 23 460-479 64 64 64 123 123 123 F F F
74792024-02-27T21:46:52.679Z 24 480-499 68 68 64 131 131 123 F F F <---
74802024-02-27T21:46:52.679Z 25 500-519 68 68 62 130 130 120 F F F <---
74812024-02-27T21:46:52.686Z 26 520-539 68 68 56 130 130 111 F F F <---
74822024-02-27T21:46:52.686Z 27 540-559 64 64 64 122 122 122 F F F
74832024-02-27T21:46:52.686Z 28 560-579 64 64 64 122 122 122 T T F <---
74842024-02-27T21:46:52.686Z 29 580-599 67 67 67 128 128 128 T T F <---
74852024-02-27T21:46:52.686ZMax gen: 68, Max flush: 131
74862024-02-27T21:46:52.686ZError: Difference in extent metadata found!
74872024-02-27T21:46:52.687ZA Difference in extent metadata is expected here
74882024-02-27T21:46:52.687ZOn loop 34
74892024-02-27T21:46:52.687Z
74902024-02-27T21:46:52.687ZVerifying data now
74912024-02-27T21:46:57.326ZNo change after verify
74922024-02-27T21:46:57.326ZLoop: 34 Downstairs dump after verify (and repair):
74932024-02-27T21:46:57.342ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74942024-02-27T21:46:57.342Z 0 000-019 64 64 64 122 122 122 F F F
74952024-02-27T21:46:57.342Z 1 020-039 64 64 64 124 124 124 F F F
74962024-02-27T21:46:57.342Z 2 040-059 69 69 69 132 132 132 F F F
74972024-02-27T21:46:57.342Z 3 060-079 61 61 61 117 117 117 F F F
74982024-02-27T21:46:57.342Z 4 080-099 68 68 68 131 131 131 F F F
74992024-02-27T21:46:57.342Z 5 100-119 68 68 68 131 131 131 F F F
75002024-02-27T21:46:57.346Z 6 120-139 68 68 68 131 131 131 F F F
75012024-02-27T21:46:57.346Z 7 140-159 67 67 67 128 128 128 F F F
75022024-02-27T21:46:57.346Z 8 160-179 67 67 67 128 128 128 F F F
75032024-02-27T21:46:57.346Z 9 180-199 68 68 68 131 131 131 F F F
75042024-02-27T21:46:57.347Z 10 200-219 68 68 68 131 131 131 F F F
75052024-02-27T21:46:57.347Z 11 220-239 67 67 67 128 128 128 F F F
75062024-02-27T21:46:57.347Z 12 240-259 63 63 63 121 121 121 F F F
75072024-02-27T21:46:57.347Z 13 260-279 64 64 64 122 122 122 F F F
75082024-02-27T21:46:57.347Z 14 280-299 68 68 68 131 131 131 F F F
75092024-02-27T21:46:57.347Z 15 300-319 64 64 64 124 124 124 F F F
75102024-02-27T21:46:57.347Z 16 320-339 68 68 68 130 130 130 F F F
75112024-02-27T21:46:57.347Z 17 340-359 65 65 65 125 125 125 F F F
75122024-02-27T21:46:57.347Z 18 360-379 58 58 58 113 113 113 F F F
75132024-02-27T21:46:57.347Z 19 380-399 67 67 67 128 128 128 F F F
75142024-02-27T21:46:57.347Z 20 400-419 67 67 67 128 128 128 F F F
75152024-02-27T21:46:57.347Z 21 420-439 69 69 69 132 132 132 F F F
75162024-02-27T21:46:57.347Z 22 440-459 63 63 63 121 121 121 F F F
75172024-02-27T21:46:57.347Z 23 460-479 64 64 64 123 123 123 F F F
75182024-02-27T21:46:57.347Z 24 480-499 68 68 68 131 131 131 F F F
75192024-02-27T21:46:57.347Z 25 500-519 68 68 68 130 130 130 F F F
75202024-02-27T21:46:57.347Z 26 520-539 68 68 68 130 130 130 F F F
75212024-02-27T21:46:57.347Z 27 540-559 64 64 64 122 122 122 F F F
75222024-02-27T21:46:57.347Z 28 560-579 69 69 69 132 132 132 F F F
75232024-02-27T21:46:57.347Z 29 580-599 69 69 69 132 132 132 F F F
75242024-02-27T21:46:57.347ZMax gen: 69, Max flush: 132
75252024-02-27T21:46:57.348Z
75262024-02-27T21:46:57.348ZBegin loop 35
75272024-02-27T21:46:57.348ZDownstairs to restart: 0
75282024-02-27T21:46:57.359Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753143957Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421}
75292024-02-27T21:46:57.359Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,Crucible runtime is spawned
75302024-02-27T21:46:57.359Z"time":"2024-02-27T21:46:55.753445122Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421}
75312024-02-27T21:46:57.366Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753500415Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421}
75322024-02-27T21:46:57.367Z{"msg":"Crucible a039410c-a55b-42b3-be14-c013ac7a1493 has session id: 62815598-cf69-4547-aad2-028f7fa03bd4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753533181Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75332024-02-27T21:46:57.367Z{"msg":"Upstairs opts: Upstairs UUID: a039410c-a55b-42b3-be14-c013ac7a1493, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753579646Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75342024-02-27T21:46:57.367Z{"msg":"Crucible stats registered with UUID: a039410c-a55b-42b3-be14-c013ac7a1493","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.75362618Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75352024-02-27T21:46:57.367Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753651867Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421}
75362024-02-27T21:46:57.367Z{"msg":"a039410c-a55b-42b3-be14-c013ac7a1493 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753666296Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75372024-02-27T21:46:57.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753678964Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75382024-02-27T21:46:57.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.75371504Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75392024-02-27T21:46:57.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753751775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75402024-02-27T21:46:57.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753778512Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75412024-02-27T21:46:57.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753816198Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75422024-02-27T21:46:57.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753877251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75432024-02-27T21:46:57.367Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753930034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75442024-02-27T21:46:57.367Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.75396539Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75452024-02-27T21:46:57.367Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.753992477Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75462024-02-27T21:46:57.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.754165097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75472024-02-27T21:46:57.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.754193673Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75482024-02-27T21:46:57.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.75422609Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75492024-02-27T21:46:57.367Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.754836848Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75502024-02-27T21:46:57.367Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.754873913Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75512024-02-27T21:46:57.367Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.754914058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75522024-02-27T21:46:57.367Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.754938576Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75532024-02-27T21:46:57.367Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.754970522Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75542024-02-27T21:46:57.368Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.754994579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75552024-02-27T21:46:57.368Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.755336249Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75562024-02-27T21:46:57.368Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.755622725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75572024-02-27T21:46:57.368Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.75566219Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75582024-02-27T21:46:57.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.755855158Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75592024-02-27T21:46:57.368Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.755908341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75602024-02-27T21:46:57.368Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.755933678Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75612024-02-27T21:46:57.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756139264Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75622024-02-27T21:46:57.368Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756193678Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75632024-02-27T21:46:57.368Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756240692Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75642024-02-27T21:46:57.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756273788Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75652024-02-27T21:46:57.368Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756298995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75662024-02-27T21:46:57.368Z{"msg":"[0]R flush_numbers[0..12]: [122, 124, 132, 117, 131, 131, 131, 128, 128, 131, 131, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756334621Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75672024-02-27T21:46:57.368Z{"msg":"[0]R generation[0..12]: [64, 64, 69, 61, 68, 68, 68, 67, 67, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756368527Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75682024-02-27T21:46:57.368Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756405913Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75692024-02-27T21:46:57.368Z{"msg":Wait for a query_work_queue command to finish before sending IO
75702024-02-27T21:46:57.368Z"[1]R flush_numbers[0..12]: [122, 124, 132, 117, 131, 131, 131, 128, 128, 131, 131, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756448588Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75712024-02-27T21:46:57.368Z{"msg":"[1]R generation[0..12]: [64, 64, 69, 61, 68, 68, 68, 67, 67, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756485153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75722024-02-27T21:46:57.368Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756509001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75732024-02-27T21:46:57.368Z{"msg":"[2]R flush_numbers[0..12]: [122, 124, 132, 117, 131, 131, 131, 128, 128, 131, 131, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756544976Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75742024-02-27T21:46:57.369Z{"msg":"[2]R generation[0..12]: [64, 64, 69, 61, 68, 68, 68, 67, 67, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756574943Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75752024-02-27T21:46:57.369Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756595311Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75762024-02-27T21:46:57.369Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756618038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75772024-02-27T21:46:57.369Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756632246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75782024-02-27T21:46:57.369Z{"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756653214Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75792024-02-27T21:46:57.369Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756674991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"":"downstairs","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75802024-02-27T21:46:57.369Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756711947Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75812024-02-27T21:46:57.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756735624Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"0","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75822024-02-27T21:46:57.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756775559Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"1","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75832024-02-27T21:46:57.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756800556Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"client":"2","":"downstairs","session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75842024-02-27T21:46:57.369Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756824684Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75852024-02-27T21:46:57.369Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.75685488Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75862024-02-27T21:46:57.369Z{"msg":"a039410c-a55b-42b3-be14-c013ac7a1493 is now active with session: 62815598-cf69-4547-aad2-028f7fa03bd4","v"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75872024-02-27T21:46:57.369Z:0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756906494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75882024-02-27T21:46:57.369Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756931131Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421,"session_id":"62815598-cf69-4547-aad2-028f7fa03bd4"}
75892024-02-27T21:46:57.370Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:46:55.756963417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421}
75902024-02-27T21:46:57.370ZLoading write count information from file "/tmp/test_repair_verify.data"
75912024-02-27T21:46:57.370ZRun Repair workload
75922024-02-27T21:46:57.370Z01/30 Write block 53 len 24576 data: 7 6 4 10 10 8
75932024-02-27T21:46:57.375Z02/30 Read block 361 len 32768
75942024-02-27T21:46:57.375Z03/30 Read block 105 len 4096
75952024-02-27T21:46:57.375Z04/30 Read block 531 len 24576
75962024-02-27T21:46:57.375Z05/30 Flush
75972024-02-27T21:46:57.375Z06/30 Write block 238 len 16384 data: 4 4 4 4
75982024-02-27T21:46:57.375Z07/30 Write block 287 len 20480 data: 6 6 9 9 9
75992024-02-27T21:46:57.375Z08/30 Write block 535 len 24576 data: 6 7 6 4 4 4
76002024-02-27T21:46:57.375Z09/30 Read block 28 len 8192
76012024-02-27T21:46:57.375Z10/30 Write block 323 len 4096 data: 5
76022024-02-27T21:46:57.375Z11/30 Read block 503 len 36864
76032024-02-27T21:46:57.375Z12/30 Write block 210 len 36864 data: 8 8 9 7 7 6 8 11 11
76042024-02-27T21:46:57.375Z13/30 Read block 89 len 24576
76052024-02-27T21:46:57.379Z14/30 Read block 65 len 20480
76062024-02-27T21:46:57.379Z15/30 Write block 69 len 4096 data: 6
76072024-02-27T21:46:57.379Z16/30 Read block 293 len 40960
76082024-02-27T21:46:57.379Z17/30 Write block 319 len 36864 data: 8 8 6 5 6 4 3 3 4
76092024-02-27T21:46:57.384Z18/30 Read block 190 len 4096
76102024-02-27T21:46:57.384Z19/30 Write block 45 len 4096 data: 6
76112024-02-27T21:46:57.384Z20/30 Read block 174 len 16384
76122024-02-27T21:46:57.384Z21/30 Read block 10 len 4096
76132024-02-27T21:46:57.384Z22/30 Write block 589 len 24576 data: 5 5 5 6 6 5
76142024-02-27T21:46:57.384Z23/30 Write block 34 len 16384 data: 9 7 7 4
76152024-02-27T21:46:57.384Z24/30 Read block 181 len 36864
76162024-02-27T21:46:57.388Z25/30 Write block 13 len 4096 data: 8
76172024-02-27T21:46:57.388Z26/30 Read block 475 len 12288
76182024-02-27T21:46:57.388Z27/30 Read block 10 len 32768
76192024-02-27T21:46:57.388Z28/30 Write block 580 len 4096 data: 5
76202024-02-27T21:46:57.389Z29/30 Write block 513 len 24576 data: 10 9 7 8 8 5
76212024-02-27T21:46:57.389Z30/30 Read block 100 len 4096
76222024-02-27T21:46:57.389Z----------------------------------------------------------------
76232024-02-27T21:46:57.389Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:25
76242024-02-27T21:46:57.389ZStates: Active Active Active
76252024-02-27T21:46:57.389ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76262024-02-27T21:46:57.389Z 6 Acked 1005 Write 4 Sent Done Done false
76272024-02-27T21:46:57.389Z 7 Acked 1006 Write 5 Sent Done Done false
76282024-02-27T21:46:57.389Z 8 Acked 1007 Write 6 Sent Done Done false
76292024-02-27T21:46:57.389Z 9 Acked 1008 Read 2 Sent Done Done false
76302024-02-27T21:46:57.389Z 10 Acked 1009 Write 1 Sent Done Done false
76312024-02-27T21:46:57.389Z 11 Acked 1010 Read 9 Sent Done Done false
76322024-02-27T21:46:57.389Z 12 Acked 1011 Write 9 Sent Done Done false
76332024-02-27T21:46:57.389Z 13 Acked 1012 Read 6 Sent Done Done false
76342024-02-27T21:46:57.389Z 14 Acked 1013 Read 5 Sent Done Done false
76352024-02-27T21:46:57.389Z 15 Acked 1014 Write 1 Sent Done Done false
76362024-02-27T21:46:57.389Z 16 Acked 1015 Read 10 Sent Done Done false
76372024-02-27T21:46:57.389Z 17 Acked 1016 Write 9 Sent Done Done false
76382024-02-27T21:46:57.389Z 18 Acked 1017 Read 1 Sent Done Done false
76392024-02-27T21:46:57.389Z 19 Acked 1018 Write 1 Sent Done Done false
76402024-02-27T21:46:57.389Z 20 Acked 1019 Read 4 Sent Done Done false
76412024-02-27T21:46:57.389Z 21 Acked 1020 Read 1 Sent Done Done false
76422024-02-27T21:46:57.390Z 22 Acked 1021 Write 6 Sent Done Done false
76432024-02-27T21:46:57.390Z 23 Acked 1022 Write 4 Sent Done Done false
76442024-02-27T21:46:57.390Z 24 Acked 1023 Read 9 Sent Done Done false
76452024-02-27T21:46:57.390Z 25 Acked 1024 Write 1 Sent Done Done false
76462024-02-27T21:46:57.390Z 26 Acked 1025 Read 3 Sent Done Done false
76472024-02-27T21:46:57.390Z 27 Acked 1026 Read 8 Sent Done Done false
76482024-02-27T21:46:57.390Z 28 Acked 1027 Write 1 Sent Done Done false
76492024-02-27T21:46:57.390Z 29 Acked 1028 Write 6 Sent Done Done false
76502024-02-27T21:46:57.390Z 30 Acked 1029 Read 1 Sent Done Sent false
76512024-02-27T21:46:57.390Z STATES DS:0 DS:1 DS:2 TOTAL
76522024-02-27T21:46:57.390Z New 0 0 0 0
76532024-02-27T21:46:57.390Z Sent 25 0 1 26
76542024-02-27T21:46:57.390Z Done 0 25 24 49
76552024-02-27T21:46:57.390Z Skipped 0 0 0 0
76562024-02-27T21:46:57.390Z Error 0 0 0 0
76572024-02-27T21:46:57.390ZLast Flush: 1004 1004 1004
76582024-02-27T21:46:57.390ZDownstairs last five completed: 1004 1003 1002 1001 1000
76592024-02-27T21:46:57.390ZUpstairs last five completed: 30 29 28 27 26
76602024-02-27T21:46:57.427Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:46:55.783066322Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1421}
76612024-02-27T21:46:57.428ZWrote out file "/tmp/test_repair_verify.data"
76622024-02-27T21:46:57.428Z
76632024-02-27T21:46:57.428ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76642024-02-27T21:46:57.428ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76652024-02-27T21:46:57.428Z 0 000-019 64 64 64 122 122 122 F T T <---
76662024-02-27T21:46:57.428Z 1 020-039 64 64 64 124 124 124 F T T <---
76672024-02-27T21:46:57.428Z 2 040-059 70 70 70 133 133 133 F T T <---
76682024-02-27T21:46:57.428Z 3 060-079 61 61 61 117 117 117 F T T <---
76692024-02-27T21:46:57.428Z 4 080-099 68 68 68 131 131 131 F F F
76702024-02-27T21:46:57.428Z 5 100-119 68 68 68 131 131 131 F F F
76712024-02-27T21:46:57.428Z 6 120-139 68 68 68 131 131 131 F F F
76722024-02-27T21:46:57.428Z 7 140-159 67 67 67 128 128 128 F F F
76732024-02-27T21:46:57.428Z 8 160-179 67 67 67 128 128 128 F F F
76742024-02-27T21:46:57.428Z 9 180-199 68 68 68 131 131 131 F F F
76752024-02-27T21:46:57.428Z 10 200-219 68 68 68 131 131 131 F T T <---
76762024-02-27T21:46:57.428Z 11 220-239 67 67 67 128 128 128 F T T <---
76772024-02-27T21:46:57.428Z 12 240-259 63 63 63 121 121 121 F T T <---
76782024-02-27T21:46:57.428Z 13 260-279 64 64 64 122 122 122 F F F
76792024-02-27T21:46:57.428Z 14 280-299 68 68 68 131 131 131 F T T <---
76802024-02-27T21:46:57.428Z 15 300-319 64 64 64 124 124 124 F T T <---
76812024-02-27T21:46:57.428Z 16 320-339 68 68 68 130 130 130 F T T <---
76822024-02-27T21:46:57.429Z 17 340-359 65 65 65 125 125 125 F F F
76832024-02-27T21:46:57.429Z 18 360-379 58 58 58 113 113 113 F F F
76842024-02-27T21:46:57.429Z 19 380-399 67 67 67 128 128 128 F F F
76852024-02-27T21:46:57.429Z 20 400-419 67 67 67 128 128 128 F F F
76862024-02-27T21:46:57.429Z 21 420-439 69 69 69 132 132 132 F F F
76872024-02-27T21:46:57.429Z 22 440-459 63 63 63 121 121 121 F F F
76882024-02-27T21:46:57.429Z 23 460-479 64 64 64 123 123 123 F F F
76892024-02-27T21:46:57.429Z 24 480-499 68 68 68 131 131 131 F F F
76902024-02-27T21:46:57.429Z 25 500-519 68 68 68 130 130 130 F T T <---
76912024-02-27T21:46:57.429Z 26 520-539 68 68 68 130 130 130 F T T <---
76922024-02-27T21:46:57.429Z 27 540-559 64 64 64 122 122 122 F T T <---
76932024-02-27T21:46:57.429Z 28 560-579 69 69 69 132 132 132 F F F
76942024-02-27T21:46:57.429Z 29 580-599 69 69 69 132 132 132 F T T <---
76952024-02-27T21:46:57.429ZMax gen: 70, Max flush: 133
76962024-02-27T21:46:57.429ZError: Difference in extent metadata found!
76972024-02-27T21:46:57.429ZA Difference in extent metadata is expected here
76982024-02-27T21:46:57.429ZOn loop 35
76992024-02-27T21:46:57.429Z
77002024-02-27T21:46:57.429ZVerifying data now
77012024-02-27T21:47:09.149ZNo change after verify
77022024-02-27T21:47:09.149ZLoop: 35 Downstairs dump after verify (and repair):
77032024-02-27T21:47:09.535ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77042024-02-27T21:47:09.536Z 0 000-019 71 71 71 134 134 134 F F F
77052024-02-27T21:47:09.536Z 1 020-039 71 71 71 134 134 134 F F F
77062024-02-27T21:47:09.536Z 2 040-059 71 71 71 134 134 134 F F F
77072024-02-27T21:47:09.536Z 3 060-079 71 71 71 134 134 134 F F F
77082024-02-27T21:47:09.536Z 4 080-099 68 68 68 131 131 131 F F F
77092024-02-27T21:47:09.536Z 5 100-119 68 68 68 131 131 131 F F F
77102024-02-27T21:47:09.536Z 6 120-139 68 68 68 131 131 131 F F F
77112024-02-27T21:47:09.536Z 7 140-159 67 67 67 128 128 128 F F F
77122024-02-27T21:47:09.536Z 8 160-179 67 67 67 128 128 128 F F F
77132024-02-27T21:47:09.536Z 9 180-199 68 68 68 131 131 131 F F F
77142024-02-27T21:47:09.536Z 10 200-219 71 71 71 134 134 134 F F F
77152024-02-27T21:47:09.537Z 11 220-239 71 71 71 134 134 134 F F F
77162024-02-27T21:47:09.537Z 12 240-259 71 71 71 134 134 134 F F F
77172024-02-27T21:47:09.537Z 13 260-279 64 64 64 122 122 122 F F F
77182024-02-27T21:47:09.537Z 14 280-299 71 71 71 134 134 134 F F F
77192024-02-27T21:47:09.537Z 15 300-319 71 71 71 134 134 134 F F F
77202024-02-27T21:47:09.537Z 16 320-339 71 71 71 134 134 134 F F F
77212024-02-27T21:47:09.537Z 17 340-359 65 65 65 125 125 125 F F F
77222024-02-27T21:47:09.537Z 18 360-379 58 58 58 113 113 113 F F F
77232024-02-27T21:47:09.537Z 19 380-399 67 67 67 128 128 128 F F F
77242024-02-27T21:47:09.537Z 20 400-419 67 67 67 128 128 128 F F F
77252024-02-27T21:47:09.538Z 21 420-439 69 69 69 132 132 132 F F F
77262024-02-27T21:47:09.538Z 22 440-459 63 63 63 121 121 121 F F F
77272024-02-27T21:47:09.538Z 23 460-479 64 64 64 123 123 123 F F F
77282024-02-27T21:47:09.538Z 24 480-499 68 68 68 131 131 131 F F F
77292024-02-27T21:47:09.538Z 25 500-519 71 71 71 134 134 134 F F F
77302024-02-27T21:47:09.538Z 26 520-539 71 71 71 134 134 134 F F F
77312024-02-27T21:47:09.538Z 27 540-559 71 71 71 134 134 134 F F F
77322024-02-27T21:47:09.540Z 28 560-579 69 69 69 132 132 132 F F F
77332024-02-27T21:47:09.540Z 29 580-599 71 71 71 134 134 134 F F F
77342024-02-27T21:47:09.540ZMax gen: 71, Max flush: 134
77352024-02-27T21:47:09.540Z
77362024-02-27T21:47:09.545ZBegin loop 36
77372024-02-27T21:47:09.545ZDownstairs to restart: 2
77382024-02-27T21:47:09.545Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553276658Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429}
77392024-02-27T21:47:09.545Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553547966Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429}
77402024-02-27T21:47:09.545Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553566274Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429}
77412024-02-27T21:47:09.546Z{"msg":"Crucible 78496019-1479-4dce-95a2-d53193103e07 has session id: 6b8e1d6e-e6a7-452e-b898-0a77e2e4516e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553593001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77422024-02-27T21:47:09.546Z{"msg":"Upstairs opts: Upstairs UUID: 78496019-1479-4dce-95a2-d53193103e07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553666382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77432024-02-27T21:47:09.546Z{Crucible runtime is spawned
77442024-02-27T21:47:09.546Z"msg":"Crucible stats registered with UUID: 78496019-1479-4dce-95a2-d53193103e07","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553717656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77452024-02-27T21:47:09.546Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553793537Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429}
77462024-02-27T21:47:09.546Z{"msg":"78496019-1479-4dce-95a2-d53193103e07 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553875328Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77472024-02-27T21:47:09.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553902144Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77482024-02-27T21:47:09.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553934541Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77492024-02-27T21:47:09.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553955808Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77502024-02-27T21:47:09.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553980665Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77512024-02-27T21:47:09.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.553996383Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77522024-02-27T21:47:09.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.554011161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77532024-02-27T21:47:09.547Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.554036849Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77542024-02-27T21:47:09.547Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.554124398Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77552024-02-27T21:47:09.547Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.554198479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77562024-02-27T21:47:09.547Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.554272471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77572024-02-27T21:47:09.547Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.554327994Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77582024-02-27T21:47:09.548Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.554391437Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77592024-02-27T21:47:09.548Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.555022562Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77602024-02-27T21:47:09.548Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.555086485Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77612024-02-27T21:47:09.548Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.55512095Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77622024-02-27T21:47:09.548Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.555165225Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77632024-02-27T21:47:09.548Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.555199021Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77642024-02-27T21:47:09.548Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.555237357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77652024-02-27T21:47:09.548Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.55563251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77662024-02-27T21:47:09.548Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.555682674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77672024-02-27T21:47:09.549Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.555703452Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77682024-02-27T21:47:09.549Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556178266Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77692024-02-27T21:47:09.549Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556223681Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77702024-02-27T21:47:09.549Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556240938Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77712024-02-27T21:47:09.549Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556262226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77722024-02-27T21:47:09.549Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556276234Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77732024-02-27T21:47:09.549Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556296662Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77742024-02-27T21:47:09.549Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.55630933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77752024-02-27T21:47:09.549Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556329358Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77762024-02-27T21:47:09.549Z{"msg":"[0]R flush_numbers[0..12]: [134, 134, 134, 134, 131, 131, 131, 128, 128, 131, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556344936Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77772024-02-27T21:47:09.549Z{"msg":"[0]R generation[0..12]: [71, 71, 71, 71, 68, 68, 68, 67, 67, 68, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556365564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77782024-02-27T21:47:09.549Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556380222Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77792024-02-27T21:47:09.550Z{"msg":"[1]R flush_numbers[0..12]: [134, 134, 134, 134, 131, 131, 131, 128, 128, 131, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556405839Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77802024-02-27T21:47:09.550Z{Wait for a query_work_queue command to finish before sending IO
77812024-02-27T21:47:09.550Z"msg":"[1]R generation[0..12]: [71, 71, 71, 71, 68, 68, 68, 67, 67, 68, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556444845Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77822024-02-27T21:47:09.550Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556478451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77832024-02-27T21:47:09.550Z{"msg":"[2]R flush_numbers[0..12]: [134, 134, 134, 134, 131, 131, 131, 128, 128, 131, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556496068Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77842024-02-27T21:47:09.550Z{"msg":"[2]R generation[0..12]: [71, 71, 71, 71, 68, 68, 68, 67, 67, 68, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556524595Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77852024-02-27T21:47:09.551Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.5565641Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77862024-02-27T21:47:09.551Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556592137Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77872024-02-27T21:47:09.551Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556607035Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77882024-02-27T21:47:09.551Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556638662Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77892024-02-27T21:47:09.551Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556673787Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"":"downstairs","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77902024-02-27T21:47:09.551Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556700095Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77912024-02-27T21:47:09.551Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556713033Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"0","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77922024-02-27T21:47:09.551Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556744849Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"1","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77932024-02-27T21:47:09.551Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556781165Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"client":"2","":"downstairs","session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77942024-02-27T21:47:09.551Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556806742Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77952024-02-27T21:47:09.552Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556819641Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77962024-02-27T21:47:09.552Z{"msg":"78496019-1479-4dce-95a2-d53193103e07 is now active with session: 6b8e1d6e-e6a7-452e-b898-0a77e2e4516e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556850767Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77972024-02-27T21:47:09.552Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556883533Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429,"session_id":"6b8e1d6e-e6a7-452e-b898-0a77e2e4516e"}
77982024-02-27T21:47:09.552Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:00.556899511Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429}
77992024-02-27T21:47:09.552ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78002024-02-27T21:47:09.553ZLoading write count information from file "/tmp/test_repair_verify.data"
78012024-02-27T21:47:09.553ZRun Repair workload
78022024-02-27T21:47:09.553Z01/30 Write block 475 len 12288 data: 5 5 4
78032024-02-27T21:47:09.584Z02/30 Read block 485 len 4096
78042024-02-27T21:47:09.584Z03/30 Read block 196 len 8192
78052024-02-27T21:47:09.584Z04/30 Write block 355 len 8192 data: 7 6
78062024-02-27T21:47:09.584Z05/30 Read block 584 len 12288
78072024-02-27T21:47:09.584Z06/30 Read block 84 len 8192
78082024-02-27T21:47:09.584Z07/30 Flush
78092024-02-27T21:47:09.584Z08/30 Write block 317 len 12288 data: 7 7 9
78102024-02-27T21:47:09.585Z09/30 Flush
78112024-02-27T21:47:09.585Z10/30 Write block 258 len 32768 data: 8 7 8 6 5 4 4 3
78122024-02-27T21:47:09.585Z11/30 Read block 176 len 28672
78132024-02-27T21:47:09.585Z12/30 Read block 326 len 16384
78142024-02-27T21:47:09.585Z13/30 Write block 356 len 24576 data: 7 4 5 7 7 4
78152024-02-27T21:47:09.585Z14/30 Write block 333 len 36864 data: 7 8 9 6 7 8 8 6 7
78162024-02-27T21:47:09.585Z15/30 Read block 418 len 36864
78172024-02-27T21:47:09.585Z16/30 Write block 181 len 8192 data: 10 10
78182024-02-27T21:47:09.585Z17/30 Flush
78192024-02-27T21:47:09.585Z18/30 Write block 250 len 28672 data: 6 5 7 7 5 8 8
78202024-02-27T21:47:09.585Z19/30 Read block 273 len 8192
78212024-02-27T21:47:09.585Z20/30 Write block 153 len 12288 data: 5 5 7
78222024-02-27T21:47:09.585Z21/30 Read block 209 len 4096
78232024-02-27T21:47:09.585Z22/30 Read block 312 len 24576
78242024-02-27T21:47:09.585Z23/30 Read block 109 len 28672
78252024-02-27T21:47:09.585Z24/30 Flush
78262024-02-27T21:47:09.585Z25/30 Write block 107 len 8192 data: 8 9
78272024-02-27T21:47:09.585Z26/30 Write block 107 len 16384 data: 9 10 10 9
78282024-02-27T21:47:09.585Z27/30 Write block 24 len 8192 data: 9 8
78292024-02-27T21:47:09.585Z28/30 Write block 100 len 12288 data: 6 5 5
78302024-02-27T21:47:09.585Z29/30 Read block 468 len 36864
78312024-02-27T21:47:09.585Z30/30 Write block 555 len 40960 data: 7 7 11 9 7 7 8 8 6 3
78322024-02-27T21:47:09.585Z----------------------------------------------------------------
78332024-02-27T21:47:09.585Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78342024-02-27T21:47:09.585ZStates: Active Active Active
78352024-02-27T21:47:09.586ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78362024-02-27T21:47:09.586Z 1 Acked 1000 Write 3 Done Done Done false
78372024-02-27T21:47:09.586Z 2 Acked 1001 Read 1 Done Done Done false
78382024-02-27T21:47:09.586Z 3 Acked 1002 Read 2 Done Done Done false
78392024-02-27T21:47:09.586Z 4 Acked 1003 Write 2 Done Done Done false
78402024-02-27T21:47:09.586Z 5 Acked 1004 Read 3 Done Done Sent false
78412024-02-27T21:47:09.586Z 6 Acked 1005 Read 2 Done Done Sent false
78422024-02-27T21:47:09.586Z 7 Acked 1006 Flush 0 Done Done Sent false
78432024-02-27T21:47:09.586Z 8 Acked 1007 Write 3 Done Done Sent false
78442024-02-27T21:47:09.586Z 9 Acked 1008 Flush 0 Done Done Sent false
78452024-02-27T21:47:09.586Z 10 Acked 1009 Write 8 Done Done Sent false
78462024-02-27T21:47:09.586Z 11 Acked 1010 Read 7 Done Done Sent false
78472024-02-27T21:47:09.586Z 12 Acked 1011 Read 4 Done Done Sent false
78482024-02-27T21:47:09.586Z 13 Acked 1012 Write 6 Done Done Sent false
78492024-02-27T21:47:09.586Z 14 Acked 1013 Write 9 Done Done Sent false
78502024-02-27T21:47:09.586Z 15 Acked 1014 Read 9 Done Done Sent false
78512024-02-27T21:47:09.586Z 16 Acked 1015 Write 2 Done Done Sent false
78522024-02-27T21:47:09.586Z 17 Acked 1016 Flush 0 Done Done Sent false
78532024-02-27T21:47:09.586Z 18 Acked 1017 Write 7 Done Done Sent false
78542024-02-27T21:47:09.587Z 19 Acked 1018 Read 2 Done Done Sent false
78552024-02-27T21:47:09.587Z 20 Acked 1019 Write 3 Done Done Sent false
78562024-02-27T21:47:09.587Z 21 Acked 1020 Read 1 Done Done Sent false
78572024-02-27T21:47:09.587Z 22 Acked 1021 Read 6 Done Done Sent false
78582024-02-27T21:47:09.587Z 23 Acked 1022 Read 7 Done Done Sent false
78592024-02-27T21:47:09.587Z 24 Acked 1023 Flush 0 Done Done Sent false
78602024-02-27T21:47:09.587Z 25 Acked 1024 Write 2 Done Done Sent false
78612024-02-27T21:47:09.587Z 26 Acked 1025 Write 4 Done Done Sent false
78622024-02-27T21:47:09.587Z 27 Acked 1026 Write 2 Done Done Sent false
78632024-02-27T21:47:09.587Z 28 Acked 1027 Write 3 Done Done Sent false
78642024-02-27T21:47:09.587Z 29 Acked 1028 Read 9 Done Done Sent false
78652024-02-27T21:47:09.587Z 30 Acked 1029 Write 10 Sent Sent Sent false
78662024-02-27T21:47:09.587Z STATES DS:0 DS:1 DS:2 TOTAL
78672024-02-27T21:47:09.587Z New 0 0 0 0
78682024-02-27T21:47:09.587Z Sent 1 1 26 28
78692024-02-27T21:47:09.587Z Done 29 29 4 62
78702024-02-27T21:47:09.587Z Skipped 0 0 0 0
78712024-02-27T21:47:09.587Z Error 0 0 0 0
78722024-02-27T21:47:09.587ZLast Flush: 1023 1023 0
78732024-02-27T21:47:09.587ZDownstairs last five completed:
78742024-02-27T21:47:09.587ZUpstairs last five completed: 30 29 28 27 26
78752024-02-27T21:47:09.587Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:00.586853663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1429}
78762024-02-27T21:47:09.587ZWrote out file "/tmp/test_repair_verify.data"
78772024-02-27T21:47:09.587Z
78782024-02-27T21:47:09.588ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78792024-02-27T21:47:09.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78802024-02-27T21:47:09.588Z 0 000-019 71 71 71 134 134 134 F F F
78812024-02-27T21:47:09.588Z 1 020-039 71 71 71 134 134 134 T T F <---
78822024-02-27T21:47:09.588Z 2 040-059 71 71 71 134 134 134 F F F
78832024-02-27T21:47:09.588Z 3 060-079 71 71 71 134 134 134 F F F
78842024-02-27T21:47:09.588Z 4 080-099 68 68 68 131 131 131 F F F
78852024-02-27T21:47:09.588Z 5 100-119 68 68 68 131 131 131 T T F <---
78862024-02-27T21:47:09.588Z 6 120-139 68 68 68 131 131 131 F F F
78872024-02-27T21:47:09.588Z 7 140-159 72 72 67 138 138 128 F F F <---
78882024-02-27T21:47:09.588Z 8 160-179 67 67 67 128 128 128 F F F
78892024-02-27T21:47:09.588Z 9 180-199 72 72 68 137 137 131 F F F <---
78902024-02-27T21:47:09.588Z 10 200-219 71 71 71 134 134 134 F F F
78912024-02-27T21:47:09.588Z 11 220-239 71 71 71 134 134 134 F F F
78922024-02-27T21:47:09.588Z 12 240-259 72 72 71 138 138 134 F F F <---
78932024-02-27T21:47:09.588Z 13 260-279 72 72 64 137 137 122 F F F <---
78942024-02-27T21:47:09.588Z 14 280-299 71 71 71 134 134 134 F F F
78952024-02-27T21:47:09.588Z 15 300-319 72 72 71 136 136 134 F F F <---
78962024-02-27T21:47:09.588Z 16 320-339 72 72 71 137 137 134 F F F <---
78972024-02-27T21:47:09.588Z 17 340-359 72 72 65 137 137 125 F F T <---
78982024-02-27T21:47:09.588Z 18 360-379 72 72 58 137 137 113 F F F <---
78992024-02-27T21:47:09.588Z 19 380-399 67 67 67 128 128 128 F F F
79002024-02-27T21:47:09.589Z 20 400-419 67 67 67 128 128 128 F F F
79012024-02-27T21:47:09.589Z 21 420-439 69 69 69 132 132 132 F F F
79022024-02-27T21:47:09.589Z 22 440-459 63 63 63 121 121 121 F F F
79032024-02-27T21:47:09.807Z 23 460-479 72 72 64 135 135 123 F F T <---
79042024-02-27T21:47:09.807Z 24 480-499 68 68 68 131 131 131 F F F
79052024-02-27T21:47:09.807Z 25 500-519 71 71 71 134 134 134 F F F
79062024-02-27T21:47:09.808Z 26 520-539 71 71 71 134 134 134 F F F
79072024-02-27T21:47:09.808Z 27 540-559 71 71 71 134 134 134 T T F <---
79082024-02-27T21:47:09.808Z 28 560-579 69 69 69 132 132 132 T T F <---
79092024-02-27T21:47:09.808Z 29 580-599 71 71 71 134 134 134 F F F
79102024-02-27T21:47:09.808ZMax gen: 72, Max flush: 138
79112024-02-27T21:47:09.808ZError: Difference in extent metadata found!
79122024-02-27T21:47:09.808ZA Difference in extent metadata is expected here
79132024-02-27T21:47:09.808ZOn loop 36
79142024-02-27T21:47:09.808Z
79152024-02-27T21:47:09.808ZVerifying data now
79162024-02-27T21:47:09.808ZNo change after verify
79172024-02-27T21:47:09.809ZLoop: 36 Downstairs dump after verify (and repair):
79182024-02-27T21:47:09.809ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79192024-02-27T21:47:09.809Z 0 000-019 71 71 71 134 134 134 F F F
79202024-02-27T21:47:09.809Z 1 020-039 73 73 73 139 139 139 F F F
79212024-02-27T21:47:09.809Z 2 040-059 71 71 71 134 134 134 F F F
79222024-02-27T21:47:09.809Z 3 060-079 71 71 71 134 134 134 F F F
79232024-02-27T21:47:09.809Z 4 080-099 68 68 68 131 131 131 F F F
79242024-02-27T21:47:09.809Z 5 100-119 73 73 73 139 139 139 F F F
79252024-02-27T21:47:09.809Z 6 120-139 68 68 68 131 131 131 F F F
79262024-02-27T21:47:09.809Z 7 140-159 72 72 72 138 138 138 F F F
79272024-02-27T21:47:09.810Z 8 160-179 67 67 67 128 128 128 F F F
79282024-02-27T21:47:09.810Z 9 180-199 72 72 72 137 137 137 F F F
79292024-02-27T21:47:09.810Z 10 200-219 71 71 71 134 134 134 F F F
79302024-02-27T21:47:09.810Z 11 220-239 71 71 71 134 134 134 F F F
79312024-02-27T21:47:09.810Z 12 240-259 72 72 72 138 138 138 F F F
79322024-02-27T21:47:09.810Z 13 260-279 72 72 72 137 137 137 F F F
79332024-02-27T21:47:09.810Z 14 280-299 71 71 71 134 134 134 F F F
79342024-02-27T21:47:09.810Z 15 300-319 72 72 72 136 136 136 F F F
79352024-02-27T21:47:09.810Z 16 320-339 72 72 72 137 137 137 F F F
79362024-02-27T21:47:09.810Z 17 340-359 72 72 72 137 137 137 F F F
79372024-02-27T21:47:09.810Z 18 360-379 72 72 72 137 137 137 F F F
79382024-02-27T21:47:09.811Z 19 380-399 67 67 67 128 128 128 F F F
79392024-02-27T21:47:09.811Z 20 400-419 67 67 67 128 128 128 F F F
79402024-02-27T21:47:09.811Z 21 420-439 69 69 69 132 132 132 F F F
79412024-02-27T21:47:09.811Z 22 440-459 63 63 63 121 121 121 F F F
79422024-02-27T21:47:09.811Z 23 460-479 72 72 72 135 135 135 F F F
79432024-02-27T21:47:09.811Z 24 480-499 68 68 68 131 131 131 F F F
79442024-02-27T21:47:09.811Z 25 500-519 71 71 71 134 134 134 F F F
79452024-02-27T21:47:09.811Z 26 520-539 71 71 71 134 134 134 F F F
79462024-02-27T21:47:09.812Z 27 540-559 73 73 73 139 139 139 F F F
79472024-02-27T21:47:09.812Z 28 560-579 73 73 73 139 139 139 F F F
79482024-02-27T21:47:09.812Z 29 580-599 71 71 71 134 134 134 F F F
79492024-02-27T21:47:09.812ZMax gen: 73, Max flush: 139
79502024-02-27T21:47:09.812Z
79512024-02-27T21:47:09.812ZBegin loop 37
79522024-02-27T21:47:09.812ZDownstairs to restart: 1
79532024-02-27T21:47:09.812Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.285417589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437}
79542024-02-27T21:47:09.812Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.28566335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437}
79552024-02-27T21:47:09.812Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.285694906Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437}
79562024-02-27T21:47:09.812Z{"msg":"Crucible 3c1b47ec-cd00-4af6-891a-621c61723b5a has session id: c595a140-29ba-4511-b9e3-15b49b8e60a2","v":0,"name":"crucible","level":30Crucible runtime is spawned
79572024-02-27T21:47:09.812Z,"time":"2024-02-27T21:47:05.285718734Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79582024-02-27T21:47:09.813Z{"msg":"Upstairs opts: Upstairs UUID: 3c1b47ec-cd00-4af6-891a-621c61723b5a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.285785406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79592024-02-27T21:47:09.813Z{"msg":"Crucible stats registered with UUID: 3c1b47ec-cd00-4af6-891a-621c61723b5a","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.285845508Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79602024-02-27T21:47:09.813Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.285878845Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437}
79612024-02-27T21:47:09.813Z{"msg":"3c1b47ec-cd00-4af6-891a-621c61723b5a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.285979353Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79622024-02-27T21:47:09.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.286038656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79632024-02-27T21:47:09.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.28609082Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79642024-02-27T21:47:09.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.286133235Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79652024-02-27T21:47:09.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.28634336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79662024-02-27T21:47:09.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.286406733Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79672024-02-27T21:47:09.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.286477034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79682024-02-27T21:47:09.814Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.286538157Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79692024-02-27T21:47:09.814Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.286606779Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79702024-02-27T21:47:09.814Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.286648824Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79712024-02-27T21:47:09.814Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.28668311Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79722024-02-27T21:47:09.814Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.286720176Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79732024-02-27T21:47:09.814Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.286757581Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79742024-02-27T21:47:09.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.287316895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79752024-02-27T21:47:09.814Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.28735713Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79762024-02-27T21:47:09.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.287395026Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79772024-02-27T21:47:09.814Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.287470657Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79782024-02-27T21:47:09.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.287520171Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79792024-02-27T21:47:09.814Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.287544718Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79802024-02-27T21:47:09.815Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.287969458Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79812024-02-27T21:47:09.815Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.28804287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79822024-02-27T21:47:09.815Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288076056Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79832024-02-27T21:47:09.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288490797Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79842024-02-27T21:47:09.815Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288534242Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79852024-02-27T21:47:09.815Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.28855027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79862024-02-27T21:47:09.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288610033Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79872024-02-27T21:47:09.815Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288630071Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79882024-02-27T21:47:09.815Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288658267Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79892024-02-27T21:47:09.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288676285Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79902024-02-27T21:47:09.815Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288704942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79912024-02-27T21:47:09.815Z{"msg":"[0]R flush_numbers[0..12]: [134, 139, 134, 134, 131, 139, 131, 138, 128, 137, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288727389Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79922024-02-27T21:47:09.815Z{"msg":"[0]R generation[0..12]: [71, 73, 71, 71, 68, 73, 68, 72, 67, 72, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288748247Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79932024-02-27T21:47:09.815Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288769974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79942024-02-27T21:47:09.816Z{"msg":"[1]R flush_numbers[0..12]: [134, 139, 134, 134, 131, 139, 131, 138, 128, 137, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288790912Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79952024-02-27T21:47:09.816Z{"msg":"[1]R generation[0..12]: [71, 73, 71, 71, 68, 73, 68, 72, 67, 72, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288811779Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79962024-02-27T21:47:09.816Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO
79972024-02-27T21:47:09.816Z"2024-02-27T21:47:05.288827027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79982024-02-27T21:47:09.816Z{"msg":"[2]R flush_numbers[0..12]: [134, 139, 134, 134, 131, 139, 131, 138, 128, 137, 134, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288860663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
79992024-02-27T21:47:09.816Z{"msg":"[2]R generation[0..12]: [71, 73, 71, 71, 68, 73, 68, 72, 67, 72, 71, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288882051Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80002024-02-27T21:47:09.816Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288912567Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80012024-02-27T21:47:09.816Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288926956Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80022024-02-27T21:47:09.816Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288940664Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80032024-02-27T21:47:09.902Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288960572Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80042024-02-27T21:47:09.902Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.28897449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"":"downstairs","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80052024-02-27T21:47:09.903Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.288988618Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80062024-02-27T21:47:09.903Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.289001387Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"0","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80072024-02-27T21:47:09.903Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.289014805Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"1","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80082024-02-27T21:47:09.903Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.289035683Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"client":"2","":"downstairs","session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80092024-02-27T21:47:09.903Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.28905852Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80102024-02-27T21:47:09.903Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.289072728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80112024-02-27T21:47:09.903Z{"msg":"3c1b47ec-cd00-4af6-891a-621c61723b5a is now active with session: c595a140-29ba-4511-b9e3-15b49b8e60a2","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.289093776Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80122024-02-27T21:47:09.903Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.289106714Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437,"session_id":"c595a140-29ba-4511-b9e3-15b49b8e60a2"}
80132024-02-27T21:47:09.903Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:05.289125472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437}
80142024-02-27T21:47:09.903ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80152024-02-27T21:47:09.903ZLoading write count information from file "/tmp/test_repair_verify.data"
80162024-02-27T21:47:09.903ZRun Repair workload
80172024-02-27T21:47:09.903ZError: Difference in extent metadata found!
80182024-02-27T21:47:09.903Z01/30 Write block 161 len 12288 data: 7 6 6
80192024-02-27T21:47:09.903Z02/30 Read block 568 len 40960
80202024-02-27T21:47:09.903Z03/30 Write block 296 len 36864 data: 7 8 8 6 6 6 8 6 9
80212024-02-27T21:47:09.903Z04/30 Read block 196 len 16384
80222024-02-27T21:47:09.903Z05/30 Flush
80232024-02-27T21:47:09.903Z06/30 Write block 459 len 8192 data: 6 5
80242024-02-27T21:47:09.903Z07/30 Read block 307 len 4096
80252024-02-27T21:47:09.903Z08/30 Write block 158 len 32768 data: 8 8 7 8 7 7 5 4
80262024-02-27T21:47:09.903Z09/30 Write block 310 len 40960 data: 12 12 11 8 7 5 8 8 8 10
80272024-02-27T21:47:09.903Z10/30 Write block 63 len 20480 data: 5 5 7 6 6
80282024-02-27T21:47:09.903Z11/30 Read block 72 len 28672
80292024-02-27T21:47:09.904Z12/30 Read block 330 len 20480
80302024-02-27T21:47:09.904Z13/30 Read block 221 len 28672
80312024-02-27T21:47:09.904Z14/30 Write block 176 len 28672 data: 7 7 8 6 8 11 11
80322024-02-27T21:47:09.904Z15/30 Read block 17 len 24576
80332024-02-27T21:47:09.904Z16/30 Read block 545 len 40960
80342024-02-27T21:47:09.904Z17/30 Read block 522 len 20480
80352024-02-27T21:47:09.904Z18/30 Write block 571 len 36864 data: 6 6 8 6 6 6 6 7 5
80362024-02-27T21:47:09.904Z19/30 Write block 282 len 16384 data: 4 6 5 4
80372024-02-27T21:47:09.904Z20/30 Flush
80382024-02-27T21:47:09.904Z21/30 Write block 43 len 32768 data: 8 7 7 4 4 5 5 6
80392024-02-27T21:47:09.904Z22/30 Write block 558 len 36864 data: 10 8 8 9 9 7 4 2 2
80402024-02-27T21:47:09.904Z23/30 Read block 221 len 4096
80412024-02-27T21:47:09.904Z24/30 Flush
80422024-02-27T21:47:09.904Z25/30 Write block 94 len 4096 data: 5
80432024-02-27T21:47:09.904Z26/30 Write block 233 len 12288 data: 10 9 9
80442024-02-27T21:47:09.904Z27/30 Read block 10 len 8192
80452024-02-27T21:47:09.904Z28/30 Write block 347 len 32768 data: 7 7 8 8 7 7 5 5
80462024-02-27T21:47:09.904Z29/30 Read block 526 len 24576
80472024-02-27T21:47:09.904Z30/30 Read block 221 len 28672
80482024-02-27T21:47:09.904Z----------------------------------------------------------------
80492024-02-27T21:47:09.904Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
80502024-02-27T21:47:09.904ZStates: Active Active Active
80512024-02-27T21:47:09.904ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80522024-02-27T21:47:09.904Z 1 Acked 1000 Write 3 Done Done Done false
80532024-02-27T21:47:09.904Z 2 Acked 1001 Read 10 Done Sent Done false
80542024-02-27T21:47:09.905Z 3 Acked 1002 Write 9 Done Sent Done false
80552024-02-27T21:47:09.905Z 4 Acked 1003 Read 4 Done Sent Done false
80562024-02-27T21:47:09.905Z 5 Acked 1004 Flush 0 Done Sent Done false
80572024-02-27T21:47:09.905Z 6 Acked 1005 Write 2 Done Sent Done false
80582024-02-27T21:47:09.905Z 7 Acked 1006 Read 1 Done Sent Done false
80592024-02-27T21:47:09.905Z 8 Acked 1007 Write 8 Done Sent Done false
80602024-02-27T21:47:09.905Z 9 Acked 1008 Write 10 Done Sent Done false
80612024-02-27T21:47:09.905Z 10 Acked 1009 Write 5 Done Sent Done false
80622024-02-27T21:47:09.905Z 11 Acked 1010 Read 7 Done Sent Done false
80632024-02-27T21:47:09.905Z 12 Acked 1011 Read 5 Done Sent Done false
80642024-02-27T21:47:09.905Z 13 Acked 1012 Read 7 Done Sent Done false
80652024-02-27T21:47:09.905Z 14 Acked 1013 Write 7 Done Sent Done false
80662024-02-27T21:47:09.905Z 15 Acked 1014 Read 6 Done Sent Done false
80672024-02-27T21:47:09.905Z 16 Acked 1015 Read 10 Done Sent Done false
80682024-02-27T21:47:09.905Z 17 Acked 1016 Read 5 Done Sent Done false
80692024-02-27T21:47:09.905Z 18 Acked 1017 Write 9 Done Sent Done false
80702024-02-27T21:47:09.905Z 19 Acked 1018 Write 4 Done Sent Done false
80712024-02-27T21:47:09.905Z 20 Acked 1019 Flush 0 Done Sent Done false
80722024-02-27T21:47:09.905Z 21 Acked 1020 Write 8 Done Sent Done false
80732024-02-27T21:47:09.905Z 22 Acked 1021 Write 9 Done Sent Done false
80742024-02-27T21:47:09.905Z 23 Acked 1022 Read 1 Done Sent Done false
80752024-02-27T21:47:09.905Z 24 Acked 1023 Flush 0 Done Sent Done false
80762024-02-27T21:47:09.905Z 25 Acked 1024 Write 1 Done Sent Done false
80772024-02-27T21:47:09.905Z 26 Acked 1025 Write 3 Done Sent Done false
80782024-02-27T21:47:09.905Z 27 Acked 1026 Read 2 Done Sent Done false
80792024-02-27T21:47:09.906Z 28 Acked 1027 Write 8 Done Sent Done false
80802024-02-27T21:47:09.906Z 29 Acked 1028 Read 6 Done Sent Done false
80812024-02-27T21:47:09.906Z 30 Acked 1029 Read 7 Done Sent Done false
80822024-02-27T21:47:09.906Z STATES DS:0 DS:1 DS:2 TOTAL
80832024-02-27T21:47:09.906Z New 0 0 0 0
80842024-02-27T21:47:09.906Z Sent 0 29 0 29
80852024-02-27T21:47:09.906Z Done 30 1 30 61
80862024-02-27T21:47:09.906Z Skipped 0 0 0 0
80872024-02-27T21:47:09.906Z Error 0 0 0 0
80882024-02-27T21:47:09.906ZLast Flush: 1023 0 1023
80892024-02-27T21:47:09.906ZDownstairs last five completed:
80902024-02-27T21:47:09.906ZUpstairs last five completed: 30 29 28 27 26
80912024-02-27T21:47:09.906Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:05.322062982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1437}
80922024-02-27T21:47:09.906ZWrote out file "/tmp/test_repair_verify.data"
80932024-02-27T21:47:09.906Z
80942024-02-27T21:47:09.906ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80952024-02-27T21:47:09.906ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80962024-02-27T21:47:09.906Z 0 000-019 71 71 71 134 134 134 F F F
80972024-02-27T21:47:09.906Z 1 020-039 73 73 73 139 139 139 F F F
80982024-02-27T21:47:09.906Z 2 040-059 74 71 74 142 134 142 F F F <---
80992024-02-27T21:47:09.906Z 3 060-079 74 71 74 141 134 141 F F F <---
81002024-02-27T21:47:09.906Z 4 080-099 68 68 68 131 131 131 T F T <---
81012024-02-27T21:47:09.906Z 5 100-119 73 73 73 139 139 139 F F F
81022024-02-27T21:47:09.906Z 6 120-139 68 68 68 131 131 131 F F F
81032024-02-27T21:47:09.951Z 7 140-159 74 72 74 141 138 141 F F F <---
81042024-02-27T21:47:09.951Z 8 160-179 74 67 74 141 128 141 F T F <---
81052024-02-27T21:47:09.951Z 9 180-199 74 72 74 141 137 141 F F F <---
81062024-02-27T21:47:09.951Z 10 200-219 71 71 71 134 134 134 F F F
81072024-02-27T21:47:09.951Z 11 220-239 71 71 71 134 134 134 T F T <---
81082024-02-27T21:47:09.951Z 12 240-259 72 72 72 138 138 138 F F F
81092024-02-27T21:47:09.951Z 13 260-279 72 72 72 137 137 137 F F F
81102024-02-27T21:47:09.951Z 14 280-299 74 71 74 141 134 141 F F F <---
81112024-02-27T21:47:09.951Z 15 300-319 74 72 74 141 136 141 F F F <---
81122024-02-27T21:47:09.951Z 16 320-339 72 72 72 137 137 137 F F F
81132024-02-27T21:47:09.951Z 17 340-359 72 72 72 137 137 137 T F T <---
81142024-02-27T21:47:09.951Z 18 360-379 72 72 72 137 137 137 F F F
81152024-02-27T21:47:09.951Z 19 380-399 67 67 67 128 128 128 F F F
81162024-02-27T21:47:09.951Z 20 400-419 67 67 67 128 128 128 F F F
81172024-02-27T21:47:09.951Z 21 420-439 69 69 69 132 132 132 F F F
81182024-02-27T21:47:09.952Z 22 440-459 74 63 74 141 121 141 F F F <---
81192024-02-27T21:47:09.952Z 23 460-479 74 72 74 141 135 141 F F F <---
81202024-02-27T21:47:09.952Z 24 480-499 68 68 68 131 131 131 F F F
81212024-02-27T21:47:09.952Z 25 500-519 71 71 71 134 134 134 F F F
81222024-02-27T21:47:09.952Z 26 520-539 71 71 71 134 134 134 F F F
81232024-02-27T21:47:09.952Z 27 540-559 74 73 74 142 139 142 F F F <---
81242024-02-27T21:47:09.952Z 28 560-579 74 73 74 142 139 142 F F F <---
81252024-02-27T21:47:09.952Z 29 580-599 71 71 71 134 134 134 F F F
81262024-02-27T21:47:09.952ZMax gen: 74, Max flush: 142
81272024-02-27T21:47:09.952ZA Difference in extent metadata is expected here
81282024-02-27T21:47:09.952ZOn loop 37
81292024-02-27T21:47:09.952Z
81302024-02-27T21:47:09.952ZVerifying data now
81312024-02-27T21:47:11.626ZNo change after verify
81322024-02-27T21:47:11.626ZLoop: 37 Downstairs dump after verify (and repair):
81332024-02-27T21:47:11.642ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81342024-02-27T21:47:11.642Z 0 000-019 71 71 71 134 134 134 F F F
81352024-02-27T21:47:11.642Z 1 020-039 73 73 73 139 139 139 F F F
81362024-02-27T21:47:11.643Z 2 040-059 74 74 74 142 142 142 F F F
81372024-02-27T21:47:11.647Z 3 060-079 74 74 74 141 141 141 F F F
81382024-02-27T21:47:11.647Z 4 080-099 75 75 75 143 143 143 F F F
81392024-02-27T21:47:11.648Z 5 100-119 73 73 73 139 139 139 F F F
81402024-02-27T21:47:11.648Z 6 120-139 68 68 68 131 131 131 F F F
81412024-02-27T21:47:11.648Z 7 140-159 74 74 74 141 141 141 F F F
81422024-02-27T21:47:11.648Z 8 160-179 74 74 74 141 141 141 F F F
81432024-02-27T21:47:11.648Z 9 180-199 74 74 74 141 141 141 F F F
81442024-02-27T21:47:11.648Z 10 200-219 71 71 71 134 134 134 F F F
81452024-02-27T21:47:11.648Z 11 220-239 75 75 75 143 143 143 F F F
81462024-02-27T21:47:11.649Z 12 240-259 72 72 72 138 138 138 F F F
81472024-02-27T21:47:11.649Z 13 260-279 72 72 72 137 137 137 F F F
81482024-02-27T21:47:11.649Z 14 280-299 74 74 74 141 141 141 F F F
81492024-02-27T21:47:11.649Z 15 300-319 74 74 74 141 141 141 F F F
81502024-02-27T21:47:11.649Z 16 320-339 72 72 72 137 137 137 F F F
81512024-02-27T21:47:11.649Z 17 340-359 75 75 75 143 143 143 F F F
81522024-02-27T21:47:11.649Z 18 360-379 72 72 72 137 137 137 F F F
81532024-02-27T21:47:11.650Z 19 380-399 67 67 67 128 128 128 F F F
81542024-02-27T21:47:11.650Z 20 400-419 67 67 67 128 128 128 F F F
81552024-02-27T21:47:11.650Z 21 420-439 69 69 69 132 132 132 F F F
81562024-02-27T21:47:11.650Z 22 440-459 74 74 74 141 141 141 F F F
81572024-02-27T21:47:11.650Z 23 460-479 74 74 74 141 141 141 F F F
81582024-02-27T21:47:11.650Z 24 480-499 68 68 68 131 131 131 F F F
81592024-02-27T21:47:11.651Z 25 500-519 71 71 71 134 134 134 F F F
81602024-02-27T21:47:11.651Z 26 520-539 71 71 71 134 134 134 F F F
81612024-02-27T21:47:11.651Z 27 540-559 74 74 74 142 142 142 F F F
81622024-02-27T21:47:11.651Z 28 560-579 74 74 74 142 142 142 F F F
81632024-02-27T21:47:11.651Z 29 580-599 71 71 71 134 134 134 F F F
81642024-02-27T21:47:11.651ZMax gen: 75, Max flush: 143
81652024-02-27T21:47:11.651Z
81662024-02-27T21:47:11.651ZBegin loop 38
81672024-02-27T21:47:11.651ZDownstairs to restart: 0
81682024-02-27T21:47:11.662Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.053855543Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445}
81692024-02-27T21:47:11.662Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.05421761Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445}
81702024-02-27T21:47:11.662Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
81712024-02-27T21:47:11.663Z,"time":"2024-02-27T21:47:10.054253846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445}
81722024-02-27T21:47:11.670Z{"msg":"Crucible 68e4b163-9f52-4e9d-ab7b-858a659a89df has session id: 2150308b-22c8-45e7-91df-d590c98dc85c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054284102Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81732024-02-27T21:47:11.670Z{"msg":"Upstairs opts: Upstairs UUID: 68e4b163-9f52-4e9d-ab7b-858a659a89df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054349165Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81742024-02-27T21:47:11.670Z{"msg":"Crucible stats registered with UUID: 68e4b163-9f52-4e9d-ab7b-858a659a89df","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.05438537Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81752024-02-27T21:47:11.670Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054422636Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445}
81762024-02-27T21:47:11.671Z{"msg":"68e4b163-9f52-4e9d-ab7b-858a659a89df active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054438394Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81772024-02-27T21:47:11.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.05447429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81782024-02-27T21:47:11.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054507336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81792024-02-27T21:47:11.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054525364Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81802024-02-27T21:47:11.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054543262Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81812024-02-27T21:47:11.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054568359Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81822024-02-27T21:47:11.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054585797Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81832024-02-27T21:47:11.671Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054610544Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81842024-02-27T21:47:11.671Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054633101Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81852024-02-27T21:47:11.671Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054663068Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81862024-02-27T21:47:11.672Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054791313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81872024-02-27T21:47:11.672Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.054999018Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81882024-02-27T21:47:11.672Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.055076499Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81892024-02-27T21:47:11.672Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.055485601Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81902024-02-27T21:47:11.672Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.055525856Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81912024-02-27T21:47:11.672Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.055616965Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81922024-02-27T21:47:11.672Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.055669069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81932024-02-27T21:47:11.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.055691156Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81942024-02-27T21:47:11.673Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.055718523Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81952024-02-27T21:47:11.673Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.056095189Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81962024-02-27T21:47:11.673Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.056161121Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81972024-02-27T21:47:11.673Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.056357498Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81982024-02-27T21:47:11.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.056572783Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
81992024-02-27T21:47:11.673Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.056611878Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82002024-02-27T21:47:11.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.056638395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82012024-02-27T21:47:11.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.05667936Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82022024-02-27T21:47:11.674Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.056715756Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82032024-02-27T21:47:11.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.056747962Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82042024-02-27T21:47:11.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.056879267Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82052024-02-27T21:47:11.674Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.05693341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82062024-02-27T21:47:11.674Z{"msg":"[0]R flush_numbers[0..12]: [134, 139, 142, 141, 143, 139, 131, 141, 141, 141, 134, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057004912Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82072024-02-27T21:47:11.675Z{"msg":"[0]R generation[0..12]: [71, 73, 74, 74, 75, 73, 68, 74, 74, 74, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.05702262Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82082024-02-27T21:47:11.675Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057050267Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82092024-02-27T21:47:11.675Z{"msg":"[1]R flush_numbers[0..12]: [134, 139, 142, 141, 143, 139, 131, 141, 141, 141, 134, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057065705Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82102024-02-27T21:47:11.675ZWait for a query_work_queue command to finish before sending IO
82112024-02-27T21:47:11.675Z{"msg":"[1]R generation[0..12]: [71, 73, 74, 74, 75, 73, 68, 74, 74, 74, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057098431Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82122024-02-27T21:47:11.675Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057115739Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82132024-02-27T21:47:11.675Z{"msg":"[2]R flush_numbers[0..12]: [134, 139, 142, 141, 143, 139, 131, 141, 141, 141, 134, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057136706Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82142024-02-27T21:47:11.675Z{"msg":"[2]R generation[0..12]: [71, 73, 74, 74, 75, 73, 68, 74, 74, 74, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057151395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82152024-02-27T21:47:11.676Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057166213Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82162024-02-27T21:47:11.676Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.0571858Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82172024-02-27T21:47:11.676Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057199759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82182024-02-27T21:47:11.676Z{"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057213687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82192024-02-27T21:47:11.676Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057234415Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"":"downstairs","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82202024-02-27T21:47:11.676Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057265521Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82212024-02-27T21:47:11.676Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057294088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"0","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82222024-02-27T21:47:11.676Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057328434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"1","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82232024-02-27T21:47:11.677Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.05736058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"client":"2","":"downstairs","session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82242024-02-27T21:47:11.677Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057393656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82252024-02-27T21:47:11.677Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057423392Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82262024-02-27T21:47:11.677Z{"msg":"68e4b163-9f52-4e9d-ab7b-858a659a89df is now active with session: 2150308b-22c8-45e7-91df-d590c98dc85c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057455709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82272024-02-27T21:47:11.677Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057486105Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445,"session_id":"2150308b-22c8-45e7-91df-d590c98dc85c"}
82282024-02-27T21:47:11.677Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:10.057515182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445}
82292024-02-27T21:47:11.677ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82302024-02-27T21:47:11.677ZLoading write count information from file "/tmp/test_repair_verify.data"
82312024-02-27T21:47:11.678ZRun Repair workload
82322024-02-27T21:47:11.678Z01/30 Write block 470 len 36864 data: 4 4 3 4 4 6 6 5 5
82332024-02-27T21:47:11.686Z02/30 Read block 68 len 4096
82342024-02-27T21:47:11.686Z03/30 Read block 268 len 8192
82352024-02-27T21:47:11.686Z04/30 Write block 549 len 24576 data: 7 6 8 6 7 8
82362024-02-27T21:47:11.686Z05/30 Read block 438 len 8192
82372024-02-27T21:47:11.686Z06/30 Write block 483 len 36864 data: 9 10 10 10 9 9 11 9 9
82382024-02-27T21:47:11.686Z07/30 Read block 193 len 32768
82392024-02-27T21:47:11.686Z08/30 Read block 420 len 20480
82402024-02-27T21:47:11.687Z09/30 Write block 320 len 24576 data: 9 7 6 7 5 4
82412024-02-27T21:47:11.687Z10/30 Read block 536 len 4096
82422024-02-27T21:47:11.687Z11/30 Write block 268 len 28672 data: 5 5 5 6 6 4 4
82432024-02-27T21:47:11.687Z12/30 Write block 481 len 28672 data: 8 8 10 11 11 11 10
82442024-02-27T21:47:11.687Z13/30 Write block 563 len 28672 data: 8 5 3 3 2 4 5
82452024-02-27T21:47:11.687Z14/30 Write block 513 len 4096 data: 11
82462024-02-27T21:47:11.687Z15/30 Write block 45 len 4096 data: 8
82472024-02-27T21:47:11.687Z16/30 Read block 225 len 4096
82482024-02-27T21:47:11.687Z17/30 Read block 267 len 4096
82492024-02-27T21:47:11.687Z18/30 Read block 356 len 40960
82502024-02-27T21:47:11.688Z19/30 Read block 244 len 20480
82512024-02-27T21:47:11.688Z20/30 Write block 138 len 32768 data: 5 6 6 6 6 6 5 4
82522024-02-27T21:47:11.688Z21/30 Write block 324 len 8192 data: 6 5
82532024-02-27T21:47:11.688Z22/30 Read block 330 len 16384
82542024-02-27T21:47:11.688Z23/30 Read block 432 len 20480
82552024-02-27T21:47:11.688Z24/30 Read block 490 len 40960
82562024-02-27T21:47:11.693Z25/30 Read block 418 len 36864
82572024-02-27T21:47:11.693Z26/30 Write block 569 len 36864 data: 6 5 7 7 9 7 7 7 7
82582024-02-27T21:47:11.693Z27/30 Write block 6 len 24576 data: 5 5 5 8 9 9
82592024-02-27T21:47:11.693Z28/30 Read block 253 len 40960
82602024-02-27T21:47:11.694Z29/30 Write block 40 len 32768 data: 8 7 7 9 8 9 5 5
82612024-02-27T21:47:11.694Z30/30 Read block 418 len 40960
82622024-02-27T21:47:11.694Z----------------------------------------------------------------
82632024-02-27T21:47:11.694Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
82642024-02-27T21:47:11.694ZStates: Active Active Active
82652024-02-27T21:47:11.694ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82662024-02-27T21:47:11.694Z 1 Acked 1000 Write 9 Done Done Done false
82672024-02-27T21:47:11.695Z 2 Acked 1001 Read 1 Done Done Done false
82682024-02-27T21:47:11.695Z 3 Acked 1002 Read 2 Done Done Done false
82692024-02-27T21:47:11.695Z 4 Acked 1003 Write 6 Done Done Done false
82702024-02-27T21:47:11.695Z 5 Acked 1004 Read 2 Done Done Done false
82712024-02-27T21:47:11.695Z 6 Acked 1005 Write 9 Done Done Done false
82722024-02-27T21:47:11.695Z 7 Acked 1006 Read 8 Done Done Done false
82732024-02-27T21:47:11.695Z 8 Acked 1007 Read 5 Done Done Done false
82742024-02-27T21:47:11.696Z 9 Acked 1008 Write 6 Done Done Done false
82752024-02-27T21:47:11.696Z 10 Acked 1009 Read 1 Done Done Done false
82762024-02-27T21:47:11.696Z 11 Acked 1010 Write 7 Done Done Done false
82772024-02-27T21:47:11.696Z 12 Acked 1011 Write 7 Done Done Done false
82782024-02-27T21:47:11.696Z 13 Acked 1012 Write 7 Done Done Done false
82792024-02-27T21:47:11.696Z 14 Acked 1013 Write 1 Sent Done Done false
82802024-02-27T21:47:11.697Z 15 Acked 1014 Write 1 Sent Done Done false
82812024-02-27T21:47:11.697Z 16 Acked 1015 Read 1 Sent Done Done false
82822024-02-27T21:47:11.697Z 17 Acked 1016 Read 1 Sent Done Done false
82832024-02-27T21:47:11.697Z 18 Acked 1017 Read 10 Sent Done Done false
82842024-02-27T21:47:11.697Z 19 Acked 1018 Read 5 Sent Done Done false
82852024-02-27T21:47:11.697Z 20 Acked 1019 Write 8 Sent Done Done false
82862024-02-27T21:47:11.697Z 21 Acked 1020 Write 2 Sent Done Done false
82872024-02-27T21:47:11.697Z 22 Acked 1021 Read 4 Sent Done Done false
82882024-02-27T21:47:11.697Z 23 Acked 1022 Read 5 Sent Done Done false
82892024-02-27T21:47:11.698Z 24 Acked 1023 Read 10 Sent Done Done false
82902024-02-27T21:47:11.698Z 25 Acked 1024 Read 9 Sent Done Done false
82912024-02-27T21:47:11.698Z 26 Acked 1025 Write 9 Sent Done Done false
82922024-02-27T21:47:11.698Z 27 Acked 1026 Write 6 Sent Done Done false
82932024-02-27T21:47:11.698Z 28 Acked 1027 Read 10 Sent Done Done false
82942024-02-27T21:47:11.698Z 29 Acked 1028 Write 8 Sent Done Done false
82952024-02-27T21:47:11.698Z 30 Acked 1029 Read 10 Sent Sent Done false
82962024-02-27T21:47:11.698Z STATES DS:0 DS:1 DS:2 TOTAL
82972024-02-27T21:47:11.699Z New 0 0 0 0
82982024-02-27T21:47:11.699Z Sent 17 1 0 18
82992024-02-27T21:47:11.699Z Done 13 29 30 72
83002024-02-27T21:47:11.699Z Skipped 0 0 0 0
83012024-02-27T21:47:11.699Z Error 0 0 0 0
83022024-02-27T21:47:11.699ZLast Flush: 0 0 0
83032024-02-27T21:47:11.699ZDownstairs last five completed:
83042024-02-27T21:47:11.700ZUpstairs last five completed: 30 29 28 27 26
83052024-02-27T21:47:11.700Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:10.084704039Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1445}
83062024-02-27T21:47:11.707ZWrote out file "/tmp/test_repair_verify.data"
83072024-02-27T21:47:11.707Z
83082024-02-27T21:47:11.707ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83092024-02-27T21:47:11.735ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83102024-02-27T21:47:11.735Z 0 000-019 71 71 71 134 134 134 F T T <---
83112024-02-27T21:47:11.736Z 1 020-039 73 73 73 139 139 139 F F F
83122024-02-27T21:47:11.736Z 2 040-059 74 74 74 142 142 142 F T T <---
83132024-02-27T21:47:11.736Z 3 060-079 74 74 74 141 141 141 F F F
83142024-02-27T21:47:11.736Z 4 080-099 75 75 75 143 143 143 F F F
83152024-02-27T21:47:11.736Z 5 100-119 73 73 73 139 139 139 F F F
83162024-02-27T21:47:11.742Z 6 120-139 68 68 68 131 131 131 F T T <---
83172024-02-27T21:47:11.742Z 7 140-159 74 74 74 141 141 141 F T T <---
83182024-02-27T21:47:11.742Z 8 160-179 74 74 74 141 141 141 F F F
83192024-02-27T21:47:11.743Z 9 180-199 74 74 74 141 141 141 F F F
83202024-02-27T21:47:11.743Z 10 200-219 71 71 71 134 134 134 F F F
83212024-02-27T21:47:11.743Z 11 220-239 75 75 75 143 143 143 F F F
83222024-02-27T21:47:11.743Z 12 240-259 72 72 72 138 138 138 F F F
83232024-02-27T21:47:11.743Z 13 260-279 72 72 72 137 137 137 T T T
83242024-02-27T21:47:11.743Z 14 280-299 74 74 74 141 141 141 F F F
83252024-02-27T21:47:11.743Z 15 300-319 74 74 74 141 141 141 F F F
83262024-02-27T21:47:11.744Z 16 320-339 72 72 72 137 137 137 T T T
83272024-02-27T21:47:11.744Z 17 340-359 75 75 75 143 143 143 F F F
83282024-02-27T21:47:11.744Z 18 360-379 72 72 72 137 137 137 F F F
83292024-02-27T21:47:11.744Z 19 380-399 67 67 67 128 128 128 F F F
83302024-02-27T21:47:11.744Z 20 400-419 67 67 67 128 128 128 F F F
83312024-02-27T21:47:11.744Z 21 420-439 69 69 69 132 132 132 F F F
83322024-02-27T21:47:11.744Z 22 440-459 74 74 74 141 141 141 F F F
83332024-02-27T21:47:11.744Z 23 460-479 74 74 74 141 141 141 T T T
83342024-02-27T21:47:11.745Z 24 480-499 68 68 68 131 131 131 T T T
83352024-02-27T21:47:11.745Z 25 500-519 71 71 71 134 134 134 F T T <---
83362024-02-27T21:47:11.745Z 26 520-539 71 71 71 134 134 134 F F F
83372024-02-27T21:47:11.745Z 27 540-559 74 74 74 142 142 142 T T T
83382024-02-27T21:47:11.745Z 28 560-579 74 74 74 142 142 142 T T T
83392024-02-27T21:47:11.745Z 29 580-599 71 71 71 134 134 134 F F F
83402024-02-27T21:47:11.745ZMax gen: 75, Max flush: 143
83412024-02-27T21:47:11.745ZError: Difference in extent metadata found!
83422024-02-27T21:47:11.746ZA Difference in extent metadata is expected here
83432024-02-27T21:47:11.746ZOn loop 38
83442024-02-27T21:47:11.746Z
83452024-02-27T21:47:11.746ZVerifying data now
83462024-02-27T21:47:17.387ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
83472024-02-27T21:47:17.392Z326a327
83482024-02-27T21:47:17.392Z> 6,
83492024-02-27T21:47:17.392Z328d328
83502024-02-27T21:47:17.392Z< 4,
83512024-02-27T21:47:17.392Z572,573d571
83522024-02-27T21:47:17.393Z< 5,
83532024-02-27T21:47:17.393Z< 4,
83542024-02-27T21:47:17.393Z575,580c573
83552024-02-27T21:47:17.393Z< 6,
83562024-02-27T21:47:17.393Z< 8,
83572024-02-27T21:47:17.393Z< 6,
83582024-02-27T21:47:17.394Z< 6,
83592024-02-27T21:47:17.394Z< 6,
83602024-02-27T21:47:17.394Z< 6,
83612024-02-27T21:47:17.394Z---
83622024-02-27T21:47:17.394Z> 5,
83632024-02-27T21:47:17.394Z581a575,581
83642024-02-27T21:47:17.394Z> 7,
83652024-02-27T21:47:17.394Z> 9,
83662024-02-27T21:47:17.394Z> 7,
83672024-02-27T21:47:17.395Z> 7,
83682024-02-27T21:47:17.395Z> 7,
83692024-02-27T21:47:17.395Z> 7,
83702024-02-27T21:47:17.395Z> 7,
83712024-02-27T21:47:17.395Zdiff found after verify
83722024-02-27T21:47:17.401ZLoop: 38 Downstairs dump after verify (and repair):
83732024-02-27T21:47:17.409ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83742024-02-27T21:47:17.409Z 0 000-019 76 76 76 144 144 144 F F F
83752024-02-27T21:47:17.409Z 1 020-039 73 73 73 139 139 139 F F F
83762024-02-27T21:47:17.409Z 2 040-059 76 76 76 144 144 144 F F F
83772024-02-27T21:47:17.409Z 3 060-079 74 74 74 141 141 141 F F F
83782024-02-27T21:47:17.409Z 4 080-099 75 75 75 143 143 143 F F F
83792024-02-27T21:47:17.414Z 5 100-119 73 73 73 139 139 139 F F F
83802024-02-27T21:47:17.414Z 6 120-139 76 76 76 144 144 144 F F F
83812024-02-27T21:47:17.414Z 7 140-159 76 76 76 144 144 144 F F F
83822024-02-27T21:47:17.414Z 8 160-179 74 74 74 141 141 141 F F F
83832024-02-27T21:47:17.414Z 9 180-199 74 74 74 141 141 141 F F F
83842024-02-27T21:47:17.414Z 10 200-219 71 71 71 134 134 134 F F F
83852024-02-27T21:47:17.415Z 11 220-239 75 75 75 143 143 143 F F F
83862024-02-27T21:47:17.415Z 12 240-259 72 72 72 138 138 138 F F F
83872024-02-27T21:47:17.415Z 13 260-279 76 76 76 144 144 144 F F F
83882024-02-27T21:47:17.415Z 14 280-299 74 74 74 141 141 141 F F F
83892024-02-27T21:47:17.416Z 15 300-319 74 74 74 141 141 141 F F F
83902024-02-27T21:47:17.416Z 16 320-339 76 76 76 144 144 144 F F F
83912024-02-27T21:47:17.416Z 17 340-359 75 75 75 143 143 143 F F F
83922024-02-27T21:47:17.416Z 18 360-379 72 72 72 137 137 137 F F F
83932024-02-27T21:47:17.416Z 19 380-399 67 67 67 128 128 128 F F F
83942024-02-27T21:47:17.416Z 20 400-419 67 67 67 128 128 128 F F F
83952024-02-27T21:47:17.416Z 21 420-439 69 69 69 132 132 132 F F F
83962024-02-27T21:47:17.417Z 22 440-459 74 74 74 141 141 141 F F F
83972024-02-27T21:47:17.417Z 23 460-479 76 76 76 144 144 144 F F F
83982024-02-27T21:47:17.417Z 24 480-499 76 76 76 144 144 144 F F F
83992024-02-27T21:47:17.417Z 25 500-519 76 76 76 144 144 144 F F F
84002024-02-27T21:47:17.417Z 26 520-539 71 71 71 134 134 134 F F F
84012024-02-27T21:47:17.417Z 27 540-559 76 76 76 144 144 144 F F F
84022024-02-27T21:47:17.417Z 28 560-579 76 76 76 144 144 144 F F F
84032024-02-27T21:47:17.417Z 29 580-599 71 71 71 134 134 134 F F F
84042024-02-27T21:47:17.417ZMax gen: 76, Max flush: 144
84052024-02-27T21:47:17.418Z
84062024-02-27T21:47:17.418ZBegin loop 39
84072024-02-27T21:47:17.418ZDownstairs to restart: 2
84082024-02-27T21:47:17.434Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821076758Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456}
84092024-02-27T21:47:17.434Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\nCrucible runtime is spawned
84102024-02-27T21:47:17.434Z rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821382652Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456}
84112024-02-27T21:47:17.447Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821417918Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456}
84122024-02-27T21:47:17.447Z{"msg":"Crucible 29d32321-c247-44e9-9c07-1066900b6309 has session id: e2d052f3-6346-4771-b4d9-4086411391fc","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821459823Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84132024-02-27T21:47:17.447Z{"msg":"Upstairs opts: Upstairs UUID: 29d32321-c247-44e9-9c07-1066900b6309, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821504008Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84142024-02-27T21:47:17.447Z{"msg":"Crucible stats registered with UUID: 29d32321-c247-44e9-9c07-1066900b6309","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821535704Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84152024-02-27T21:47:17.447Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821559351Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456}
84162024-02-27T21:47:17.447Z{"msg":"29d32321-c247-44e9-9c07-1066900b6309 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821585338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84172024-02-27T21:47:17.447Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821607586Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84182024-02-27T21:47:17.448Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.82165248Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84192024-02-27T21:47:17.448Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821679117Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84202024-02-27T21:47:17.448Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821711723Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84212024-02-27T21:47:17.448Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821744369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84222024-02-27T21:47:17.448Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821809032Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84232024-02-27T21:47:17.448Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821874064Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84242024-02-27T21:47:17.448Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.821917709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84252024-02-27T21:47:17.448Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.822043664Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84262024-02-27T21:47:17.448Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.822110076Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84272024-02-27T21:47:17.448Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.822200986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84282024-02-27T21:47:17.448Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.822233162Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84292024-02-27T21:47:17.448Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.822794606Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84302024-02-27T21:47:17.448Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.822833991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84312024-02-27T21:47:17.448Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.822863578Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84322024-02-27T21:47:17.448Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.822931539Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84332024-02-27T21:47:17.448Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.823376947Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84342024-02-27T21:47:17.448Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.823448189Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84352024-02-27T21:47:17.448Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.823495313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84362024-02-27T21:47:17.448Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.823533189Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84372024-02-27T21:47:17.448Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.823954049Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84382024-02-27T21:47:17.448Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.823997114Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84392024-02-27T21:47:17.449Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824013782Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84402024-02-27T21:47:17.449Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824110401Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84412024-02-27T21:47:17.449Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824137248Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84422024-02-27T21:47:17.449Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824152026Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84432024-02-27T21:47:17.449Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824170284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84442024-02-27T21:47:17.449Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824567287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84452024-02-27T21:47:17.449Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824598363Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84462024-02-27T21:47:17.449Z{"msg":"[0]R flush_numbers[0..12]: [144, 139, 144, 141, 143, 139, 144, 144, 141, 141, 134, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824661346Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84472024-02-27T21:47:17.449Z{"msg":"[0]R generation[0..12]: [76, 73, 76, 74, 75, 73, 76, 76, 74, 74, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824687153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84482024-02-27T21:47:17.449Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824702001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84492024-02-27T21:47:17.449Z{"msg":"[1]R flush_numbers[0..12]: [144, 139, 144, 141, 143, 139, 144, 144, 141, 141, 134, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824723539Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84502024-02-27T21:47:17.449Z{"msg":"[1]R generation[0..12]: [76, 73, 76, 74, 75, 73, 76, 76, 74, 74, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824737657Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84512024-02-27T21:47:17.449Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824752775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84522024-02-27T21:47:17.449Z{"msg":"[2]R flush_numbers[0..12]: [144, 139, 144, 141, 143, 139, 144, 144, 141, 141, 134, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824767203Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84532024-02-27T21:47:17.449Z{"msg":"[2]R generation[0..12]: [76, 73, 76, 74, 75, 73, 76, 76, 74, 74, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824782021Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84542024-02-27T21:47:17.449ZWait for a query_work_queue command to finish before sending IO
84552024-02-27T21:47:17.449Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824800969Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84562024-02-27T21:47:17.449Z{"msg":"Max found gen is 77","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824821717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84572024-02-27T21:47:17.449Z{"msg":"Generation requested: 78 >= found:77","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824835805Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84582024-02-27T21:47:17.449Z{"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824849154Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84592024-02-27T21:47:17.449Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824862322Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"":"downstairs","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84602024-02-27T21:47:17.449Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824875881Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84612024-02-27T21:47:17.449Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824895678Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"0","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84622024-02-27T21:47:17.449Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824909726Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"1","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84632024-02-27T21:47:17.449Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824929764Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"client":"2","":"downstairs","session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84642024-02-27T21:47:17.450Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824943422Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84652024-02-27T21:47:17.450Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824955821Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84662024-02-27T21:47:17.450Z{"msg":"29d32321-c247-44e9-9c07-1066900b6309 is now active with session: e2d052f3-6346-4771-b4d9-4086411391fc","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824977369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84672024-02-27T21:47:17.450Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.824990067Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456,"session_id":"e2d052f3-6346-4771-b4d9-4086411391fc"}
84682024-02-27T21:47:17.450Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:15.825003246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456}
84692024-02-27T21:47:17.450ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84702024-02-27T21:47:17.450ZLoading write count information from file "/tmp/test_repair_verify.data"
84712024-02-27T21:47:17.450ZRun Repair workload
84722024-02-27T21:47:17.450Z01/30 Write block 62 len 12288 data: 4 6 6
84732024-02-27T21:47:17.450Z02/30 Read block 194 len 36864
84742024-02-27T21:47:17.450Z03/30 Write block 394 len 8192 data: 4 4
84752024-02-27T21:47:17.450Z04/30 Write block 142 len 8192 data: 7 7
84762024-02-27T21:47:17.450Z05/30 Write block 552 len 40960 data: 7 8 9 8 8 12 11 9 9 10
84772024-02-27T21:47:17.462Z06/30 Read block 45 len 40960
84782024-02-27T21:47:17.462Z07/30 Read block 49 len 4096
84792024-02-27T21:47:17.462Z08/30 Write block 330 len 28672 data: 6 6 7 8 9 10 7
84802024-02-27T21:47:17.462Z09/30 Write block 9 len 20480 data: 9 10 10 9 9
84812024-02-27T21:47:17.462Z10/30 Read block 80 len 16384
84822024-02-27T21:47:17.462Z11/30 Read block 434 len 20480
84832024-02-27T21:47:17.463Z12/30 Read block 128 len 4096
84842024-02-27T21:47:17.463Z13/30 Read block 173 len 32768
84852024-02-27T21:47:17.463Z14/30 Write block 466 len 32768 data: 8 7 7 6 5 5 4 5
84862024-02-27T21:47:17.463Z15/30 Read block 9 len 24576
84872024-02-27T21:47:17.463Z16/30 Read block 162 len 32768
84882024-02-27T21:47:17.463Z17/30 Write block 99 len 20480 data: 6 7 6 6 5
84892024-02-27T21:47:17.463Z18/30 Read block 398 len 8192
84902024-02-27T21:47:17.463Z19/30 Flush
84912024-02-27T21:47:17.464Z20/30 Write block 340 len 12288 data: 7 8 8
84922024-02-27T21:47:17.464Z21/30 Write block 480 len 4096 data: 7
84932024-02-27T21:47:17.468Z22/30 Read block 270 len 12288
84942024-02-27T21:47:17.468Z23/30 Write block 400 len 36864 data: 8 9 6 5 6 5 5 4 5
84952024-02-27T21:47:17.468Z24/30 Read block 217 len 16384
84962024-02-27T21:47:17.468Z25/30 Write block 415 len 40960 data: 5 6 8 7 7 4 3 3 3 4
84972024-02-27T21:47:17.468Z26/30 Read block 532 len 32768
84982024-02-27T21:47:17.468Z27/30 Write block 467 len 12288 data: 8 8 7
84992024-02-27T21:47:17.469Z28/30 Read block 546 len 32768
85002024-02-27T21:47:17.469Z29/30 Read block 83 len 40960
85012024-02-27T21:47:17.469Z30/30 Write block 174 len 12288 data: 8 7 8
85022024-02-27T21:47:17.469Z----------------------------------------------------------------
85032024-02-27T21:47:17.469Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
85042024-02-27T21:47:17.469ZStates: Active Active Active
85052024-02-27T21:47:17.469ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
85062024-02-27T21:47:17.469Z 1 Acked 1000 Write 3 Done Done Done false
85072024-02-27T21:47:17.469Z 2 Acked 1001 Read 9 Done Done Sent false
85082024-02-27T21:47:17.469Z 3 Acked 1002 Write 2 Done Done Sent false
85092024-02-27T21:47:17.469Z 4 Acked 1003 Write 2 Done Done Sent false
85102024-02-27T21:47:17.470Z 5 Acked 1004 Write 10 Done Done Sent false
85112024-02-27T21:47:17.470Z 6 Acked 1005 Read 10 Done Done Sent false
85122024-02-27T21:47:17.470Z 7 Acked 1006 Read 1 Done Done Sent false
85132024-02-27T21:47:17.470Z 8 Acked 1007 Write 7 Done Done Sent false
85142024-02-27T21:47:17.470Z 9 Acked 1008 Write 5 Done Done Sent false
85152024-02-27T21:47:17.470Z 10 Acked 1009 Read 4 Done Done Sent false
85162024-02-27T21:47:17.470Z 11 Acked 1010 Read 5 Done Done Sent false
85172024-02-27T21:47:17.470Z 12 Acked 1011 Read 1 Done Done Sent false
85182024-02-27T21:47:17.470Z 13 Acked 1012 Read 8 Done Done Sent false
85192024-02-27T21:47:17.470Z 14 Acked 1013 Write 8 Done Done Sent false
85202024-02-27T21:47:17.470Z 15 Acked 1014 Read 6 Done Done Sent false
85212024-02-27T21:47:17.470Z 16 Acked 1015 Read 8 Done Done Sent false
85222024-02-27T21:47:17.470Z 17 Acked 1016 Write 5 Done Done Sent false
85232024-02-27T21:47:17.470Z 18 Acked 1017 Read 2 Done Done Sent false
85242024-02-27T21:47:17.470Z 19 Acked 1018 Flush 0 Done Done Sent false
85252024-02-27T21:47:17.470Z 20 Acked 1019 Write 3 Done Done Sent false
85262024-02-27T21:47:17.470Z 21 Acked 1020 Write 1 Done Done Sent false
85272024-02-27T21:47:17.470Z 22 Acked 1021 Read 3 Done Done Sent false
85282024-02-27T21:47:17.470Z 23 Acked 1022 Write 9 Done Done Sent false
85292024-02-27T21:47:17.470Z 24 Acked 1023 Read 4 Done Done Sent false
85302024-02-27T21:47:17.470Z 25 Acked 1024 Write 10 Done Done Sent false
85312024-02-27T21:47:17.470Z 26 Acked 1025 Read 8 Done Done Sent false
85322024-02-27T21:47:17.470Z 27 Acked 1026 Write 3 Done Done Sent false
85332024-02-27T21:47:17.470Z 28 Acked 1027 Read 8 Done Done Sent false
85342024-02-27T21:47:17.470Z 29 Acked 1028 Read 10 Done Done Sent false
85352024-02-27T21:47:17.470Z 30 Acked 1029 Write 3 Sent Sent Sent false
85362024-02-27T21:47:17.471Z STATES DS:0 DS:1 DS:2 TOTAL
85372024-02-27T21:47:17.471Z New 0 0 0 0
85382024-02-27T21:47:17.471Z Sent 1 1 29 31
85392024-02-27T21:47:17.471Z Done 29 29 1 59
85402024-02-27T21:47:17.471Z Skipped 0 0 0 0
85412024-02-27T21:47:17.471Z Error 0 0 0 0
85422024-02-27T21:47:17.471ZLast Flush: 1018 1018 0
85432024-02-27T21:47:17.471ZDownstairs last five completed:
85442024-02-27T21:47:17.471ZUpstairs last five completed: 30 29 28 27 26
85452024-02-27T21:47:17.471Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:15.854543686Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1456}
85462024-02-27T21:47:17.471ZWrote out file "/tmp/test_repair_verify.data"
85472024-02-27T21:47:17.471Z
85482024-02-27T21:47:17.475ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85492024-02-27T21:47:17.487ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85502024-02-27T21:47:17.487Z 0 000-019 78 78 76 145 145 144 F F F <---
85512024-02-27T21:47:17.487Z 1 020-039 73 73 73 139 139 139 F F F
85522024-02-27T21:47:17.487Z 2 040-059 76 76 76 144 144 144 F F F
85532024-02-27T21:47:17.495Z 3 060-079 78 78 74 145 145 141 F F T <---
85542024-02-27T21:47:17.495Z 4 080-099 78 78 75 145 145 143 F F F <---
85552024-02-27T21:47:17.495Z 5 100-119 78 78 73 145 145 139 F F F <---
85562024-02-27T21:47:17.495Z 6 120-139 76 76 76 144 144 144 F F F
85572024-02-27T21:47:17.495Z 7 140-159 78 78 76 145 145 144 F F F <---
85582024-02-27T21:47:17.495Z 8 160-179 74 74 74 141 141 141 T T F <---
85592024-02-27T21:47:17.495Z 9 180-199 74 74 74 141 141 141 F F F
85602024-02-27T21:47:17.495Z 10 200-219 71 71 71 134 134 134 F F F
85612024-02-27T21:47:17.495Z 11 220-239 75 75 75 143 143 143 F F F
85622024-02-27T21:47:17.495Z 12 240-259 72 72 72 138 138 138 F F F
85632024-02-27T21:47:17.495Z 13 260-279 76 76 76 144 144 144 F F F
85642024-02-27T21:47:17.495Z 14 280-299 74 74 74 141 141 141 F F F
85652024-02-27T21:47:17.495Z 15 300-319 74 74 74 141 141 141 F F F
85662024-02-27T21:47:17.495Z 16 320-339 78 78 76 145 145 144 F F F <---
85672024-02-27T21:47:17.495Z 17 340-359 75 75 75 143 143 143 T T F <---
85682024-02-27T21:47:17.495Z 18 360-379 72 72 72 137 137 137 F F F
85692024-02-27T21:47:17.495Z 19 380-399 78 78 67 145 145 128 F F F <---
85702024-02-27T21:47:17.495Z 20 400-419 67 67 67 128 128 128 T T F <---
85712024-02-27T21:47:17.495Z 21 420-439 69 69 69 132 132 132 T T F <---
85722024-02-27T21:47:17.496Z 22 440-459 74 74 74 141 141 141 F F F
85732024-02-27T21:47:17.496Z 23 460-479 78 78 76 145 145 144 T T F <---
85742024-02-27T21:47:17.496Z 24 480-499 76 76 76 144 144 144 T T F <---
85752024-02-27T21:47:17.496Z 25 500-519 76 76 76 144 144 144 F F F
85762024-02-27T21:47:17.496Z 26 520-539 71 71 71 134 134 134 F F F
85772024-02-27T21:47:17.496Z 27 540-559 78 78 76 145 145 144 F F F <---
85782024-02-27T21:47:17.496Z 28 560-579 78 78 76 145 145 144 F F F <---
85792024-02-27T21:47:17.496Z 29 580-599 71 71 71 134 134 134 F F F
85802024-02-27T21:47:17.496ZMax gen: 78, Max flush: 145
85812024-02-27T21:47:17.496ZError: Difference in extent metadata found!
85822024-02-27T21:47:17.496ZA Difference in extent metadata is expected here
85832024-02-27T21:47:17.496ZOn loop 39
85842024-02-27T21:47:17.496Z
85852024-02-27T21:47:17.507ZVerifying data now
85862024-02-27T21:47:22.247ZNo change after verify
85872024-02-27T21:47:22.247ZLoop: 39 Downstairs dump after verify (and repair):
85882024-02-27T21:47:22.262ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85892024-02-27T21:47:22.262Z 0 000-019 78 78 78 145 145 145 F F F
85902024-02-27T21:47:22.262Z 1 020-039 73 73 73 139 139 139 F F F
85912024-02-27T21:47:22.262Z 2 040-059 76 76 76 144 144 144 F F F
85922024-02-27T21:47:22.263Z 3 060-079 78 78 78 145 145 145 F F F
85932024-02-27T21:47:22.263Z 4 080-099 78 78 78 145 145 145 F F F
85942024-02-27T21:47:22.263Z 5 100-119 78 78 78 145 145 145 F F F
85952024-02-27T21:47:22.263Z 6 120-139 76 76 76 144 144 144 F F F
85962024-02-27T21:47:22.263Z 7 140-159 78 78 78 145 145 145 F F F
85972024-02-27T21:47:22.263Z 8 160-179 79 79 79 146 146 146 F F F
85982024-02-27T21:47:22.268Z 9 180-199 74 74 74 141 141 141 F F F
85992024-02-27T21:47:22.268Z 10 200-219 71 71 71 134 134 134 F F F
86002024-02-27T21:47:22.268Z 11 220-239 75 75 75 143 143 143 F F F
86012024-02-27T21:47:22.269Z 12 240-259 72 72 72 138 138 138 F F F
86022024-02-27T21:47:22.269Z 13 260-279 76 76 76 144 144 144 F F F
86032024-02-27T21:47:22.269Z 14 280-299 74 74 74 141 141 141 F F F
86042024-02-27T21:47:22.269Z 15 300-319 74 74 74 141 141 141 F F F
86052024-02-27T21:47:22.269Z 16 320-339 78 78 78 145 145 145 F F F
86062024-02-27T21:47:22.270Z 17 340-359 79 79 79 146 146 146 F F F
86072024-02-27T21:47:22.270Z 18 360-379 72 72 72 137 137 137 F F F
86082024-02-27T21:47:22.270Z 19 380-399 78 78 78 145 145 145 F F F
86092024-02-27T21:47:22.270Z 20 400-419 79 79 79 146 146 146 F F F
86102024-02-27T21:47:22.270Z 21 420-439 79 79 79 146 146 146 F F F
86112024-02-27T21:47:22.270Z 22 440-459 74 74 74 141 141 141 F F F
86122024-02-27T21:47:22.271Z 23 460-479 79 79 79 146 146 146 F F F
86132024-02-27T21:47:22.271Z 24 480-499 79 79 79 146 146 146 F F F
86142024-02-27T21:47:22.271Z 25 500-519 76 76 76 144 144 144 F F F
86152024-02-27T21:47:22.271Z 26 520-539 71 71 71 134 134 134 F F F
86162024-02-27T21:47:22.271Z 27 540-559 78 78 78 145 145 145 F F F
86172024-02-27T21:47:22.272Z 28 560-579 78 78 78 145 145 145 F F F
86182024-02-27T21:47:22.272Z 29 580-599 71 71 71 134 134 134 F F F
86192024-02-27T21:47:22.272ZMax gen: 79, Max flush: 146
86202024-02-27T21:47:22.272Z
86212024-02-27T21:47:22.272ZBegin loop 40
86222024-02-27T21:47:22.272ZDownstairs to restart: 0
86232024-02-27T21:47:22.291Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.674511179Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464}
86242024-02-27T21:47:22.291Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
86252024-02-27T21:47:22.291Z,"time":"2024-02-27T21:47:20.674856009Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464}
86262024-02-27T21:47:22.299Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.674889185Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464}
86272024-02-27T21:47:22.299Z{"msg":"Crucible aa609cad-199b-4fa5-a5d2-754035f097af has session id: bc2f662e-9535-4bd6-befc-fa7849a8d306","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.67492641Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86282024-02-27T21:47:22.299Z{"msg":"Upstairs opts: Upstairs UUID: aa609cad-199b-4fa5-a5d2-754035f097af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.674970055Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86292024-02-27T21:47:22.299Z{"msg":"Crucible stats registered with UUID: aa609cad-199b-4fa5-a5d2-754035f097af","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675003491Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86302024-02-27T21:47:22.299Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675028038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464}
86312024-02-27T21:47:22.300Z{"msg":"aa609cad-199b-4fa5-a5d2-754035f097af active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675053066Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86322024-02-27T21:47:22.300Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675074103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86332024-02-27T21:47:22.300Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675119958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86342024-02-27T21:47:22.300Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675149254Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86352024-02-27T21:47:22.300Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675176131Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86362024-02-27T21:47:22.300Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675204178Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86372024-02-27T21:47:22.300Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675239733Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86382024-02-27T21:47:22.300Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.67526619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86392024-02-27T21:47:22.301Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675294507Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86402024-02-27T21:47:22.301Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675320124Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86412024-02-27T21:47:22.301Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675345351Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86422024-02-27T21:47:22.301Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675371578Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86432024-02-27T21:47:22.301Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.675411773Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86442024-02-27T21:47:22.301Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.67603442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86452024-02-27T21:47:22.301Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.676081065Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86462024-02-27T21:47:22.302Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.676108671Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86472024-02-27T21:47:22.302Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.676167894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86482024-02-27T21:47:22.302Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.676606263Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86492024-02-27T21:47:22.302Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.676644318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86502024-02-27T21:47:22.302Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.676748336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86512024-02-27T21:47:22.302Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.676777802Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86522024-02-27T21:47:22.302Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677173216Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86532024-02-27T21:47:22.302Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677338316Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86542024-02-27T21:47:22.303Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677376082Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86552024-02-27T21:47:22.303Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.67739251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86562024-02-27T21:47:22.303Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677407288Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86572024-02-27T21:47:22.303Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677442744Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86582024-02-27T21:47:22.303Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677457522Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86592024-02-27T21:47:22.303Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677633702Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86602024-02-27T21:47:22.303Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677698394Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86612024-02-27T21:47:22.303Z{"msg":"[0]R flush_numbers[0..12]: [145, 139, 144, 145, 145, 145, 144, 145, 146, 141, 134, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677745338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86622024-02-27T21:47:22.304Z{"msg":"[0]R generation[0..12]: [78, 73, 76, 78, 78, 78, 76, 78, 79, 74, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677783484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86632024-02-27T21:47:22.304Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":Wait for a query_work_queue command to finish before sending IO
86642024-02-27T21:47:22.304Z0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677825009Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86652024-02-27T21:47:22.304Z{"msg":"[1]R flush_numbers[0..12]: [145, 139, 144, 145, 145, 145, 144, 145, 146, 141, 134, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677868574Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86662024-02-27T21:47:22.304Z{"msg":"[1]R generation[0..12]: [78, 73, 76, 78, 78, 78, 76, 78, 79, 74, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677908389Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86672024-02-27T21:47:22.304Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.677950444Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86682024-02-27T21:47:22.304Z{"msg":"[2]R flush_numbers[0..12]: [145, 139, 144, 145, 145, 145, 144, 145, 146, 141, 134, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.67798951Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86692024-02-27T21:47:22.305Z{"msg":"[2]R generation[0..12]: [78, 73, 76, 78, 78, 78, 76, 78, 79, 74, 71, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678020066Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86702024-02-27T21:47:22.305Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678059671Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86712024-02-27T21:47:22.305Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678097607Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86722024-02-27T21:47:22.305Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86732024-02-27T21:47:22.305Z,"time":"2024-02-27T21:47:20.678134803Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86742024-02-27T21:47:22.305Z{"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678178477Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86752024-02-27T21:47:22.306Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678207344Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"":"downstairs","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86762024-02-27T21:47:22.306Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678222472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86772024-02-27T21:47:22.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.67824134Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"0","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86782024-02-27T21:47:22.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678255498Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"1","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86792024-02-27T21:47:22.306Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678269337Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"client":"2","":"downstairs","session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86802024-02-27T21:47:22.306Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678290194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86812024-02-27T21:47:22.307Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678302983Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86822024-02-27T21:47:22.307Z{"msg":"aa609cad-199b-4fa5-a5d2-754035f097af is now active with session: bc2f662e-9535-4bd6-befc-fa7849a8d306","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678322261Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86832024-02-27T21:47:22.307Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678334739Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464,"session_id":"bc2f662e-9535-4bd6-befc-fa7849a8d306"}
86842024-02-27T21:47:22.307Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:20.678347967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464}
86852024-02-27T21:47:22.307ZLoading write count information from file "/tmp/test_repair_verify.data"
86862024-02-27T21:47:22.307ZRun Repair workload
86872024-02-27T21:47:22.308Z01/30 Write block 236 len 4096 data: 9
86882024-02-27T21:47:22.308Z02/30 Read block 73 len 36864
86892024-02-27T21:47:22.308Z03/30 Read block 256 len 8192
86902024-02-27T21:47:22.308Z04/30 Read block 303 len 4096
86912024-02-27T21:47:22.309Z05/30 Read block 382 len 24576
86922024-02-27T21:47:22.309Z06/30 Read block 354 len 20480
86932024-02-27T21:47:22.309Z07/30 Flush
86942024-02-27T21:47:22.315Z08/30 Write block 182 len 12288 data: 12 10 9
86952024-02-27T21:47:22.315Z09/30 Read block 175 len 28672
86962024-02-27T21:47:22.315Z10/30 Read block 247 len 16384
86972024-02-27T21:47:22.315Z11/30 Read block 188 len 4096
86982024-02-27T21:47:22.315Z12/30 Write block 425 len 12288 data: 3 3 3
86992024-02-27T21:47:22.315Z13/30 Write block 15 len 28672 data: 9 9 8 6 5 5 5
87002024-02-27T21:47:22.315Z14/30 Read block 504 len 20480
87012024-02-27T21:47:22.316Z15/30 Read block 493 len 24576
87022024-02-27T21:47:22.316Z16/30 Write block 482 len 8192 data: 9 11
87032024-02-27T21:47:22.316Z17/30 Read block 240 len 12288
87042024-02-27T21:47:22.316Z18/30 Read block 504 len 36864
87052024-02-27T21:47:22.316Z19/30 Read block 46 len 40960
87062024-02-27T21:47:22.316Z20/30 Flush
87072024-02-27T21:47:22.316Z21/30 Write block 173 len 36864 data: 6 9 8 9 8 9 7 9 12
87082024-02-27T21:47:22.316Z22/30 Read block 196 len 28672
87092024-02-27T21:47:22.316Z23/30 Read block 244 len 36864
87102024-02-27T21:47:22.316Z24/30 Write block 274 len 4096 data: 5
87112024-02-27T21:47:22.316Z25/30 Write block 234 len 40960 data: 10 10 10 5 5 5 5 5 4 5
87122024-02-27T21:47:22.317Z26/30 Read block 486 len 28672
87132024-02-27T21:47:22.317Z27/30 Read block 91 len 24576
87142024-02-27T21:47:22.322Z28/30 Read block 123 len 36864
87152024-02-27T21:47:22.322Z29/30 Write block 548 len 24576 data: 7 8 7 9 8 9
87162024-02-27T21:47:22.322Z30/30 Read block 531 len 32768
87172024-02-27T21:47:22.322Z----------------------------------------------------------------
87182024-02-27T21:47:22.323Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
87192024-02-27T21:47:22.323ZStates: Active Active Active
87202024-02-27T21:47:22.323ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87212024-02-27T21:47:22.323Z 1 Acked 1000 Write 1 Done Done Done false
87222024-02-27T21:47:22.324Z 2 Acked 1001 Read 9 Done Done Done false
87232024-02-27T21:47:22.324Z 3 Acked 1002 Read 2 Done Done Done false
87242024-02-27T21:47:22.324Z 4 Acked 1003 Read 1 Sent Done Done false
87252024-02-27T21:47:22.324Z 5 Acked 1004 Read 6 Sent Done Done false
87262024-02-27T21:47:22.324Z 6 Acked 1005 Read 5 Sent Done Done false
87272024-02-27T21:47:22.324Z 7 Acked 1006 Flush 0 Sent Done Done false
87282024-02-27T21:47:22.324Z 8 Acked 1007 Write 3 Sent Done Done false
87292024-02-27T21:47:22.324Z 9 Acked 1008 Read 7 Sent Done Done false
87302024-02-27T21:47:22.324Z 10 Acked 1009 Read 4 Sent Done Done false
87312024-02-27T21:47:22.324Z 11 Acked 1010 Read 1 Sent Done Done false
87322024-02-27T21:47:22.324Z 12 Acked 1011 Write 3 Sent Done Done false
87332024-02-27T21:47:22.324Z 13 Acked 1012 Write 7 Sent Done Done false
87342024-02-27T21:47:22.324Z 14 Acked 1013 Read 5 Sent Done Done false
87352024-02-27T21:47:22.324Z 15 Acked 1014 Read 6 Sent Done Done false
87362024-02-27T21:47:22.324Z 16 Acked 1015 Write 2 Sent Done Done false
87372024-02-27T21:47:22.324Z 17 Acked 1016 Read 3 Sent Done Done false
87382024-02-27T21:47:22.325Z 18 Acked 1017 Read 9 Sent Done Done false
87392024-02-27T21:47:22.325Z 19 Acked 1018 Read 10 Sent Done Done false
87402024-02-27T21:47:22.325Z 20 Acked 1019 Flush 0 Sent Done Done false
87412024-02-27T21:47:22.325Z 21 Acked 1020 Write 9 Sent Done Done false
87422024-02-27T21:47:22.325Z 22 Acked 1021 Read 7 Sent Done Done false
87432024-02-27T21:47:22.325Z 23 Acked 1022 Read 9 Sent Done Done false
87442024-02-27T21:47:22.325Z 24 Acked 1023 Write 1 Sent Done Done false
87452024-02-27T21:47:22.325Z 25 Acked 1024 Write 10 Sent Done Done false
87462024-02-27T21:47:22.325Z 26 Acked 1025 Read 7 Sent Done Done false
87472024-02-27T21:47:22.325Z 27 Acked 1026 Read 6 Sent Done Done false
87482024-02-27T21:47:22.325Z 28 Acked 1027 Read 9 Sent Done Done false
87492024-02-27T21:47:22.325Z 29 Acked 1028 Write 6 Sent Done Done false
87502024-02-27T21:47:22.325Z 30 Acked 1029 Read 8 Sent Done Sent false
87512024-02-27T21:47:22.325Z STATES DS:0 DS:1 DS:2 TOTAL
87522024-02-27T21:47:22.325Z New 0 0 0 0
87532024-02-27T21:47:22.325Z Sent 27 0 1 28
87542024-02-27T21:47:22.325Z Done 3 30 29 62
87552024-02-27T21:47:22.325Z Skipped 0 0 0 0
87562024-02-27T21:47:22.325Z Error 0 0 0 0
87572024-02-27T21:47:22.325ZLast Flush: 0 1019 1019
87582024-02-27T21:47:22.325ZDownstairs last five completed:
87592024-02-27T21:47:22.325ZUpstairs last five completed: 30 29 28 27 26
87602024-02-27T21:47:22.325Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:20.709975892Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1464}
87612024-02-27T21:47:22.326ZWrote out file "/tmp/test_repair_verify.data"
87622024-02-27T21:47:22.330Z
87632024-02-27T21:47:22.330ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87642024-02-27T21:47:22.350ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87652024-02-27T21:47:22.350Z 0 000-019 78 80 80 145 148 148 F F F <---
87662024-02-27T21:47:22.350Z 1 020-039 73 80 80 139 148 148 F F F <---
87672024-02-27T21:47:22.350Z 2 040-059 76 76 76 144 144 144 F F F
87682024-02-27T21:47:22.350Z 3 060-079 78 78 78 145 145 145 F F F
87692024-02-27T21:47:22.350Z 4 080-099 78 78 78 145 145 145 F F F
87702024-02-27T21:47:22.350Z 5 100-119 78 78 78 145 145 145 F F F
87712024-02-27T21:47:22.351Z 6 120-139 76 76 76 144 144 144 F F F
87722024-02-27T21:47:22.351Z 7 140-159 78 78 78 145 145 145 F F F
87732024-02-27T21:47:22.351Z 8 160-179 79 79 79 146 146 146 F T T <---
87742024-02-27T21:47:22.351Z 9 180-199 74 80 80 141 148 148 F T T <---
87752024-02-27T21:47:22.351Z 10 200-219 71 71 71 134 134 134 F F F
87762024-02-27T21:47:22.355Z 11 220-239 75 80 80 143 147 147 T T T <---
87772024-02-27T21:47:22.355Z 12 240-259 72 72 72 138 138 138 F T T <---
87782024-02-27T21:47:22.355Z 13 260-279 76 76 76 144 144 144 F T T <---
87792024-02-27T21:47:22.356Z 14 280-299 74 74 74 141 141 141 F F F
87802024-02-27T21:47:22.356Z 15 300-319 74 74 74 141 141 141 F F F
87812024-02-27T21:47:22.356Z 16 320-339 78 78 78 145 145 145 F F F
87822024-02-27T21:47:22.356Z 17 340-359 79 79 79 146 146 146 F F F
87832024-02-27T21:47:22.356Z 18 360-379 72 72 72 137 137 137 F F F
87842024-02-27T21:47:22.356Z 19 380-399 78 78 78 145 145 145 F F F
87852024-02-27T21:47:22.356Z 20 400-419 79 79 79 146 146 146 F F F
87862024-02-27T21:47:22.357Z 21 420-439 79 80 80 146 148 148 F F F <---
87872024-02-27T21:47:22.357Z 22 440-459 74 74 74 141 141 141 F F F
87882024-02-27T21:47:22.357Z 23 460-479 79 79 79 146 146 146 F F F
87892024-02-27T21:47:22.357Z 24 480-499 79 80 80 146 148 148 F F F <---
87902024-02-27T21:47:22.357Z 25 500-519 76 76 76 144 144 144 F F F
87912024-02-27T21:47:22.357Z 26 520-539 71 71 71 134 134 134 F F F
87922024-02-27T21:47:22.357Z 27 540-559 78 78 78 145 145 145 F T T <---
87932024-02-27T21:47:22.357Z 28 560-579 78 78 78 145 145 145 F F F
87942024-02-27T21:47:22.357Z 29 580-599 71 71 71 134 134 134 F F F
87952024-02-27T21:47:22.358ZMax gen: 80, Max flush: 148
87962024-02-27T21:47:22.358ZError: Difference in extent metadata found!
87972024-02-27T21:47:22.358ZA Difference in extent metadata is expected here
87982024-02-27T21:47:22.358ZOn loop 40
87992024-02-27T21:47:22.358Z
88002024-02-27T21:47:22.358ZVerifying data now
88012024-02-27T21:47:26.891ZNo change after verify
88022024-02-27T21:47:26.891ZLoop: 40 Downstairs dump after verify (and repair):
88032024-02-27T21:47:26.905ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88042024-02-27T21:47:26.905Z 0 000-019 80 80 80 148 148 148 F F F
88052024-02-27T21:47:26.905Z 1 020-039 80 80 80 148 148 148 F F F
88062024-02-27T21:47:26.906Z 2 040-059 76 76 76 144 144 144 F F F
88072024-02-27T21:47:26.910Z 3 060-079 78 78 78 145 145 145 F F F
88082024-02-27T21:47:26.910Z 4 080-099 78 78 78 145 145 145 F F F
88092024-02-27T21:47:26.910Z 5 100-119 78 78 78 145 145 145 F F F
88102024-02-27T21:47:26.910Z 6 120-139 76 76 76 144 144 144 F F F
88112024-02-27T21:47:26.911Z 7 140-159 78 78 78 145 145 145 F F F
88122024-02-27T21:47:26.911Z 8 160-179 81 81 81 149 149 149 F F F
88132024-02-27T21:47:26.911Z 9 180-199 81 81 81 149 149 149 F F F
88142024-02-27T21:47:26.911Z 10 200-219 71 71 71 134 134 134 F F F
88152024-02-27T21:47:26.911Z 11 220-239 81 81 81 149 149 149 F F F
88162024-02-27T21:47:26.911Z 12 240-259 81 81 81 149 149 149 F F F
88172024-02-27T21:47:26.911Z 13 260-279 81 81 81 149 149 149 F F F
88182024-02-27T21:47:26.911Z 14 280-299 74 74 74 141 141 141 F F F
88192024-02-27T21:47:26.911Z 15 300-319 74 74 74 141 141 141 F F F
88202024-02-27T21:47:26.911Z 16 320-339 78 78 78 145 145 145 F F F
88212024-02-27T21:47:26.911Z 17 340-359 79 79 79 146 146 146 F F F
88222024-02-27T21:47:26.911Z 18 360-379 72 72 72 137 137 137 F F F
88232024-02-27T21:47:26.912Z 19 380-399 78 78 78 145 145 145 F F F
88242024-02-27T21:47:26.912Z 20 400-419 79 79 79 146 146 146 F F F
88252024-02-27T21:47:26.912Z 21 420-439 80 80 80 148 148 148 F F F
88262024-02-27T21:47:26.912Z 22 440-459 74 74 74 141 141 141 F F F
88272024-02-27T21:47:26.912Z 23 460-479 79 79 79 146 146 146 F F F
88282024-02-27T21:47:26.912Z 24 480-499 80 80 80 148 148 148 F F F
88292024-02-27T21:47:26.912Z 25 500-519 76 76 76 144 144 144 F F F
88302024-02-27T21:47:26.912Z 26 520-539 71 71 71 134 134 134 F F F
88312024-02-27T21:47:26.912Z 27 540-559 81 81 81 149 149 149 F F F
88322024-02-27T21:47:26.912Z 28 560-579 78 78 78 145 145 145 F F F
88332024-02-27T21:47:26.912Z 29 580-599 71 71 71 134 134 134 F F F
88342024-02-27T21:47:26.912ZMax gen: 81, Max flush: 149
88352024-02-27T21:47:26.912Z
88362024-02-27T21:47:26.912ZBegin loop 41
88372024-02-27T21:47:26.913ZDownstairs to restart: 1
88382024-02-27T21:47:26.927Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.316858087Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472}
88392024-02-27T21:47:26.927Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317154063Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472}
88402024-02-27T21:47:26.936Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317192358Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472}
88412024-02-27T21:47:26.936Z{"msg":"Crucible 55e0a340-705a-4994-a08a-9e133271891c has session id: 4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317216685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88422024-02-27T21:47:26.937Z{"msg":"Upstairs opts: Upstairs UUID: 55e0a340-705a-4994-a08a-9e133271891c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317249731Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id"Crucible runtime is spawned
88432024-02-27T21:47:26.937Z:"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88442024-02-27T21:47:26.937Z{"msg":"Crucible stats registered with UUID: 55e0a340-705a-4994-a08a-9e133271891c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317282447Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88452024-02-27T21:47:26.937Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317362308Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472}
88462024-02-27T21:47:26.937Z{"msg":"55e0a340-705a-4994-a08a-9e133271891c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317480894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88472024-02-27T21:47:26.937Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.31750983Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88482024-02-27T21:47:26.937Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317545966Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88492024-02-27T21:47:26.938Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317562264Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88502024-02-27T21:47:26.938Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317583742Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88512024-02-27T21:47:26.938Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.31759993Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88522024-02-27T21:47:26.938Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317617568Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88532024-02-27T21:47:26.938Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317639775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88542024-02-27T21:47:26.938Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317658673Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88552024-02-27T21:47:26.938Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317673361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88562024-02-27T21:47:26.938Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317747373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88572024-02-27T21:47:26.939Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317792267Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88582024-02-27T21:47:26.939Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.317828643Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88592024-02-27T21:47:26.939Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.318510763Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88602024-02-27T21:47:26.939Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.318538239Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88612024-02-27T21:47:26.939Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.31869873Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88622024-02-27T21:47:26.939Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.318780711Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88632024-02-27T21:47:26.939Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.318831805Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88642024-02-27T21:47:26.940Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.318862061Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88652024-02-27T21:47:26.940Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.319361223Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88662024-02-27T21:47:26.940Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.319408377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88672024-02-27T21:47:26.940Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.319456311Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88682024-02-27T21:47:26.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.319985369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88692024-02-27T21:47:26.940Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320036743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88702024-02-27T21:47:26.940Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320052471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88712024-02-27T21:47:26.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320083917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88722024-02-27T21:47:26.940Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320117663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88732024-02-27T21:47:26.940Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320150759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88742024-02-27T21:47:26.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320181856Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88752024-02-27T21:47:26.940ZWait for a query_work_queue command to finish before sending IO
88762024-02-27T21:47:26.940Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320216382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88772024-02-27T21:47:26.940Z{"msg":"[0]R flush_numbers[0..12]: [148, 148, 144, 145, 145, 145, 144, 145, 149, 149, 134, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320250318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88782024-02-27T21:47:26.940Z{"msg":"[0]R generation[0..12]: [80, 80, 76, 78, 78, 78, 76, 78, 81, 81, 71, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320271575Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88792024-02-27T21:47:26.940Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320301672Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88802024-02-27T21:47:26.940Z{"msg":"[1]R flush_numbers[0..12]: [148, 148, 144, 145, 145, 145, 144, 145, 149, 149, 134, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320333908Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88812024-02-27T21:47:26.940Z{"msg":"[1]R generation[0..12]: [80, 80, 76, 78, 78, 78, 76, 78, 81, 81, 71, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320368964Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88822024-02-27T21:47:26.940Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.32040114Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88832024-02-27T21:47:26.941Z{"msg":"[2]R flush_numbers[0..12]: [148, 148, 144, 145, 145, 145, 144, 145, 149, 149, 134, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320435226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88842024-02-27T21:47:26.941Z{"msg":"[2]R generation[0..12]: [80, 80, 76, 78, 78, 78, 76, 78, 81, 81, 71, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320455174Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88852024-02-27T21:47:26.941Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.32048517Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88862024-02-27T21:47:26.941Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320500528Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88872024-02-27T21:47:26.941Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320530695Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88882024-02-27T21:47:26.941Z{"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320552152Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88892024-02-27T21:47:26.941Z{"msg":"All extents match","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88902024-02-27T21:47:26.941Z,"time":"2024-02-27T21:47:25.320583428Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"":"downstairs","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88912024-02-27T21:47:26.941Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320617395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88922024-02-27T21:47:26.941Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320649721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"0","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88932024-02-27T21:47:26.941Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320681337Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"1","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88942024-02-27T21:47:26.941Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320712733Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"client":"2","":"downstairs","session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88952024-02-27T21:47:26.941Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320744729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88962024-02-27T21:47:26.941Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320784925Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88972024-02-27T21:47:26.941Z{"msg":"55e0a340-705a-4994-a08a-9e133271891c is now active with session: 4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320818051Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88982024-02-27T21:47:26.941Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320846777Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472,"session_id":"4b8e0cf4-c4a3-4a0a-90cd-cf21819d061d"}
88992024-02-27T21:47:26.941Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:25.320878914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472}
89002024-02-27T21:47:26.941ZLoading write count information from file "/tmp/test_repair_verify.data"
89012024-02-27T21:47:26.941ZRun Repair workload
89022024-02-27T21:47:26.941Z01/30 Write block 231 len 36864 data: 11 12 11 11 11 11 6 6 6
89032024-02-27T21:47:26.946Z02/30 Read block 505 len 40960
89042024-02-27T21:47:26.946Z03/30 Read block 540 len 40960
89052024-02-27T21:47:26.946Z04/30 Read block 253 len 40960
89062024-02-27T21:47:26.946Z05/30 Write block 317 len 24576 data: 9 9 11 10 8 7
89072024-02-27T21:47:26.947Z06/30 Read block 390 len 28672
89082024-02-27T21:47:26.947Z07/30 Read block 295 len 8192
89092024-02-27T21:47:26.947Z08/30 Flush
89102024-02-27T21:47:26.947Z09/30 Write block 355 len 4096 data: 8
89112024-02-27T21:47:26.947Z10/30 Write block 488 len 28672 data: 10 12 10 10 9 7 7
89122024-02-27T21:47:26.948Z11/30 Read block 2 len 16384
89132024-02-27T21:47:26.948Z12/30 Write block 551 len 28672 data: 10 9 10 10 9 9 13
89142024-02-27T21:47:26.948Z13/30 Write block 546 len 28672 data: 3 3 8 9 8 11 10
89152024-02-27T21:47:26.948Z14/30 Write block 373 len 40960 data: 3 2 2 2 3 3 3 3 3 3
89162024-02-27T21:47:26.952Z15/30 Write block 91 len 20480 data: 7 3 5 6 5
89172024-02-27T21:47:26.952Z16/30 Write block 184 len 24576 data: 10 6 7 7 9 8
89182024-02-27T21:47:26.953Z17/30 Flush
89192024-02-27T21:47:26.953Z18/30 Write block 30 len 36864 data: 8 8 7 7 10 8 8 5 5
89202024-02-27T21:47:26.953Z19/30 Read block 146 len 20480
89212024-02-27T21:47:26.956Z20/30 Read block 386 len 12288
89222024-02-27T21:47:26.956Z21/30 Flush
89232024-02-27T21:47:26.956Z22/30 Write block 534 len 40960 data: 6 7 8 7 5 5 5 5 5 4
89242024-02-27T21:47:26.961Z23/30 Write block 382 len 24576 data: 4 4 5 7 7 9
89252024-02-27T21:47:26.961Z24/30 Write block 340 len 4096 data: 8
89262024-02-27T21:47:26.961Z25/30 Write block 554 len 40960 data: 11 10 10 14 12 10 10 11 10 9
89272024-02-27T21:47:26.961Z26/30 Read block 389 len 8192
89282024-02-27T21:47:26.962Z27/30 Read block 71 len 40960
89292024-02-27T21:47:26.966Z28/30 Write block 5 len 32768 data: 4 6 6 6 10 11 11 10
89302024-02-27T21:47:26.967Z29/30 Read block 452 len 20480
89312024-02-27T21:47:26.967Z30/30 Read block 431 len 28672
89322024-02-27T21:47:26.967Z----------------------------------------------------------------
89332024-02-27T21:47:26.967Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:22
89342024-02-27T21:47:26.967ZStates: Active Active Active
89352024-02-27T21:47:26.968ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89362024-02-27T21:47:26.968Z 9 Acked 1008 Write 1 Done Sent Done false
89372024-02-27T21:47:26.968Z 10 Acked 1009 Write 7 Done Sent Done false
89382024-02-27T21:47:26.968Z 11 Acked 1010 Read 4 Done Sent Done false
89392024-02-27T21:47:26.968Z 12 Acked 1011 Write 7 Done Sent Done false
89402024-02-27T21:47:26.968Z 13 Acked 1012 Write 7 Done Sent Done false
89412024-02-27T21:47:26.969Z 14 Acked 1013 Write 10 Done Sent Done false
89422024-02-27T21:47:26.969Z 15 Acked 1014 Write 5 Done Sent Done false
89432024-02-27T21:47:26.969Z 16 Acked 1015 Write 6 Done Sent Done false
89442024-02-27T21:47:26.969Z 17 Acked 1016 Flush 0 Done Sent Done false
89452024-02-27T21:47:26.969Z 18 Acked 1017 Write 9 Done Sent Done false
89462024-02-27T21:47:26.969Z 19 Acked 1018 Read 5 Done Sent Done false
89472024-02-27T21:47:26.969Z 20 Acked 1019 Read 3 Done Sent Done false
89482024-02-27T21:47:26.969Z 21 Acked 1020 Flush 0 Done Sent Done false
89492024-02-27T21:47:26.969Z 22 Acked 1021 Write 10 Done Sent Done false
89502024-02-27T21:47:26.969Z 23 Acked 1022 Write 6 Done Sent Done false
89512024-02-27T21:47:26.969Z 24 Acked 1023 Write 1 Done Sent Done false
89522024-02-27T21:47:26.969Z 25 Acked 1024 Write 10 Done Sent Done false
89532024-02-27T21:47:26.969Z 26 Acked 1025 Read 2 Done Sent Done false
89542024-02-27T21:47:26.969Z 27 Acked 1026 Read 10 Done Sent Done false
89552024-02-27T21:47:26.970Z 28 Acked 1027 Write 8 Done Sent Done false
89562024-02-27T21:47:26.970Z 29 Acked 1028 Read 5 Done Sent Done false
89572024-02-27T21:47:26.970Z 30 Acked 1029 Read 7 Done Sent Sent false
89582024-02-27T21:47:26.970Z STATES DS:0 DS:1 DS:2 TOTAL
89592024-02-27T21:47:26.970Z New 0 0 0 0
89602024-02-27T21:47:26.970Z Sent 0 22 1 23
89612024-02-27T21:47:26.970Z Done 22 0 21 43
89622024-02-27T21:47:26.970Z Skipped 0 0 0 0
89632024-02-27T21:47:26.970Z Error 0 0 0 0
89642024-02-27T21:47:26.970ZLast Flush: 1020 1007 1020
89652024-02-27T21:47:26.970ZDownstairs last five completed: 1007 1006 1005 1004 1003
89662024-02-27T21:47:26.970ZUpstairs last five completed: 30 29 28 27 26
89672024-02-27T21:47:26.970Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:25.354656025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1472}
89682024-02-27T21:47:26.970ZWrote out file "/tmp/test_repair_verify.data"
89692024-02-27T21:47:26.975Z
89702024-02-27T21:47:26.975ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89712024-02-27T21:47:26.989ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89722024-02-27T21:47:26.989Z 0 000-019 80 80 80 148 148 148 T F T <---
89732024-02-27T21:47:26.989Z 1 020-039 82 80 82 152 148 152 F F F <---
89742024-02-27T21:47:26.989Z 2 040-059 76 76 76 144 144 144 F F F
89752024-02-27T21:47:26.989Z 3 060-079 78 78 78 145 145 145 F F F
89762024-02-27T21:47:26.989Z 4 080-099 82 78 82 151 145 151 F F F <---
89772024-02-27T21:47:26.990Z 5 100-119 78 78 78 145 145 145 F F F
89782024-02-27T21:47:26.990Z 6 120-139 76 76 76 144 144 144 F F F
89792024-02-27T21:47:26.994Z 7 140-159 78 78 78 145 145 145 F F F
89802024-02-27T21:47:26.994Z 8 160-179 81 81 81 149 149 149 F F F
89812024-02-27T21:47:26.994Z 9 180-199 82 81 82 151 149 151 F F F <---
89822024-02-27T21:47:26.994Z 10 200-219 71 71 71 134 134 134 F F F
89832024-02-27T21:47:26.994Z 11 220-239 82 82 82 150 150 150 F F F
89842024-02-27T21:47:26.994Z 12 240-259 81 81 81 149 149 149 F F F
89852024-02-27T21:47:26.994Z 13 260-279 81 81 81 149 149 149 F F F
89862024-02-27T21:47:26.994Z 14 280-299 74 74 74 141 141 141 F F F
89872024-02-27T21:47:26.994Z 15 300-319 82 82 82 150 150 150 F F F
89882024-02-27T21:47:26.994Z 16 320-339 82 82 82 150 150 150 F F F
89892024-02-27T21:47:26.994Z 17 340-359 82 79 82 151 146 151 T F T <---
89902024-02-27T21:47:26.995Z 18 360-379 82 72 82 151 137 151 F F F <---
89912024-02-27T21:47:26.995Z 19 380-399 82 78 82 151 145 151 T F T <---
89922024-02-27T21:47:26.995Z 20 400-419 79 79 79 146 146 146 F F F
89932024-02-27T21:47:26.995Z 21 420-439 80 80 80 148 148 148 F F F
89942024-02-27T21:47:26.995Z 22 440-459 74 74 74 141 141 141 F F F
89952024-02-27T21:47:26.995Z 23 460-479 79 79 79 146 146 146 F F F
89962024-02-27T21:47:26.995Z 24 480-499 82 80 82 151 148 151 F F F <---
89972024-02-27T21:47:26.995Z 25 500-519 76 76 76 144 144 144 F F F
89982024-02-27T21:47:26.995Z 26 520-539 71 71 71 134 134 134 T F T <---
89992024-02-27T21:47:26.995Z 27 540-559 82 81 82 151 149 151 T F T <---
90002024-02-27T21:47:26.996Z 28 560-579 78 78 78 145 145 145 T F T <---
90012024-02-27T21:47:26.996Z 29 580-599 71 71 71 134 134 134 F F F
90022024-02-27T21:47:26.996ZMax gen: 82, Max flush: 152
90032024-02-27T21:47:26.996ZError: Difference in extent metadata found!
90042024-02-27T21:47:26.996ZA Difference in extent metadata is expected here
90052024-02-27T21:47:26.996ZOn loop 41
90062024-02-27T21:47:26.996Z
90072024-02-27T21:47:27.000ZVerifying data now
90082024-02-27T21:47:31.577ZNo change after verify
90092024-02-27T21:47:31.581ZLoop: 41 Downstairs dump after verify (and repair):
90102024-02-27T21:47:31.593ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90112024-02-27T21:47:31.593Z 0 000-019 83 83 83 153 153 153 F F F
90122024-02-27T21:47:31.593Z 1 020-039 82 82 82 152 152 152 F F F
90132024-02-27T21:47:31.593Z 2 040-059 76 76 76 144 144 144 F F F
90142024-02-27T21:47:31.593Z 3 060-079 78 78 78 145 145 145 F F F
90152024-02-27T21:47:31.593Z 4 080-099 82 82 82 151 151 151 F F F
90162024-02-27T21:47:31.593Z 5 100-119 78 78 78 145 145 145 F F F
90172024-02-27T21:47:31.593Z 6 120-139 76 76 76 144 144 144 F F F
90182024-02-27T21:47:31.598Z 7 140-159 78 78 78 145 145 145 F F F
90192024-02-27T21:47:31.598Z 8 160-179 81 81 81 149 149 149 F F F
90202024-02-27T21:47:31.598Z 9 180-199 82 82 82 151 151 151 F F F
90212024-02-27T21:47:31.598Z 10 200-219 71 71 71 134 134 134 F F F
90222024-02-27T21:47:31.598Z 11 220-239 82 82 82 150 150 150 F F F
90232024-02-27T21:47:31.598Z 12 240-259 81 81 81 149 149 149 F F F
90242024-02-27T21:47:31.598Z 13 260-279 81 81 81 149 149 149 F F F
90252024-02-27T21:47:31.598Z 14 280-299 74 74 74 141 141 141 F F F
90262024-02-27T21:47:31.598Z 15 300-319 82 82 82 150 150 150 F F F
90272024-02-27T21:47:31.599Z 16 320-339 82 82 82 150 150 150 F F F
90282024-02-27T21:47:31.599Z 17 340-359 83 83 83 153 153 153 F F F
90292024-02-27T21:47:31.599Z 18 360-379 82 82 82 151 151 151 F F F
90302024-02-27T21:47:31.599Z 19 380-399 83 83 83 153 153 153 F F F
90312024-02-27T21:47:31.599Z 20 400-419 79 79 79 146 146 146 F F F
90322024-02-27T21:47:31.599Z 21 420-439 80 80 80 148 148 148 F F F
90332024-02-27T21:47:31.599Z 22 440-459 74 74 74 141 141 141 F F F
90342024-02-27T21:47:31.599Z 23 460-479 79 79 79 146 146 146 F F F
90352024-02-27T21:47:31.599Z 24 480-499 82 82 82 151 151 151 F F F
90362024-02-27T21:47:31.599Z 25 500-519 76 76 76 144 144 144 F F F
90372024-02-27T21:47:31.600Z 26 520-539 83 83 83 153 153 153 F F F
90382024-02-27T21:47:31.600Z 27 540-559 83 83 83 153 153 153 F F F
90392024-02-27T21:47:31.600Z 28 560-579 83 83 83 153 153 153 F F F
90402024-02-27T21:47:31.600Z 29 580-599 71 71 71 134 134 134 F F F
90412024-02-27T21:47:31.600ZMax gen: 83, Max flush: 153
90422024-02-27T21:47:31.600Z
90432024-02-27T21:47:31.600ZBegin loop 42
90442024-02-27T21:47:31.600ZDownstairs to restart: 0
90452024-02-27T21:47:31.617Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.00644773Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480}
90462024-02-27T21:47:31.641Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.006733926Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480}
90472024-02-27T21:47:31.641ZCrucible runtime is spawned
90482024-02-27T21:47:31.641Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.00678797Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480}
90492024-02-27T21:47:31.641Z{"msg":"Crucible bfba2fa9-5eb6-45b8-a083-f46659b623d5 has session id: 2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.006825745Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90502024-02-27T21:47:31.641Z{"msg":"Upstairs opts: Upstairs UUID: bfba2fa9-5eb6-45b8-a083-f46659b623d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.006873959Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90512024-02-27T21:47:31.641Z{"msg":"Crucible stats registered with UUID: bfba2fa9-5eb6-45b8-a083-f46659b623d5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.006909186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90522024-02-27T21:47:31.642Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.006939452Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480}
90532024-02-27T21:47:31.642Z{"msg":"bfba2fa9-5eb6-45b8-a083-f46659b623d5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007002804Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90542024-02-27T21:47:31.642Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007069787Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90552024-02-27T21:47:31.642Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007105932Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90562024-02-27T21:47:31.642Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007145208Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90572024-02-27T21:47:31.642Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007173064Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90582024-02-27T21:47:31.642Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.00720779Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90592024-02-27T21:47:31.642Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007233627Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90602024-02-27T21:47:31.642Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.00729041Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90612024-02-27T21:47:31.642Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007324597Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90622024-02-27T21:47:31.642Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007362282Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90632024-02-27T21:47:31.643Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007388589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90642024-02-27T21:47:31.643Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007425665Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90652024-02-27T21:47:31.643Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.007453921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90662024-02-27T21:47:31.643Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.008150579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90672024-02-27T21:47:31.643Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.008180426Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90682024-02-27T21:47:31.643Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.008199403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90692024-02-27T21:47:31.643Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.008243098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90702024-02-27T21:47:31.643Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.008860126Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90712024-02-27T21:47:31.643Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.008914359Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90722024-02-27T21:47:31.643Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.009052353Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90732024-02-27T21:47:31.643Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.009086459Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90742024-02-27T21:47:31.643Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.009410361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90752024-02-27T21:47:31.644Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.009457365Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90762024-02-27T21:47:31.644Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.009487662Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90772024-02-27T21:47:31.644Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.009513339Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90782024-02-27T21:47:31.644Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.009528227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90792024-02-27T21:47:31.644Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.009541815Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90802024-02-27T21:47:31.644Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.009764279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90812024-02-27T21:47:31.644Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010226635Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90822024-02-27T21:47:31.644Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010257041Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90832024-02-27T21:47:31.644Z{"msg":"[0]R flush_numbers[0..12]: [153, 152, 144, 145, 151, 145, 144, 145, 149, 151, 134, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010274929Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90842024-02-27T21:47:31.644Z{"msg":"[0]R generation[0..12]: [83, 82, 76, 78, 82, 78, 76, 78, 81, 82, 71, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010297066Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90852024-02-27T21:47:31.644Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010329823Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90862024-02-27T21:47:31.644Z{"msg":"[1]R flush_numbers[0..12]: [153, 152, 144, 145, 151, 145, 144, 145, 149, 151, 134, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.01035098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90872024-02-27T21:47:31.645Z{"msg":"[1]R generation[0..12]: [83, 82, 76, 78, 82, 78, 76, 78, 81, 82, 71, 82]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
90882024-02-27T21:47:31.645Z,"time":"2024-02-27T21:47:30.010431711Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90892024-02-27T21:47:31.650Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010473806Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90902024-02-27T21:47:31.650Z{"msg":"[2]R flush_numbers[0..12]: [153, 152, 144, 145, 151, 145, 144, 145, 149, 151, 134, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010509591Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90912024-02-27T21:47:31.650Z{"msg":"[2]R generation[0..12]: [83, 82, 76, 78, 82, 78, 76, 78, 81, 82, 71, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010538308Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90922024-02-27T21:47:31.650Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010575843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90932024-02-27T21:47:31.650Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.01061035Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90942024-02-27T21:47:31.650Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010636276Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90952024-02-27T21:47:31.651Z{"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010668463Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90962024-02-27T21:47:31.651Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.01068908Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"":"downstairs","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90972024-02-27T21:47:31.651Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010725306Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90982024-02-27T21:47:31.651Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010768231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"0","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
90992024-02-27T21:47:31.651Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010803207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"1","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
91002024-02-27T21:47:31.651Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010828034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"client":"2","":"downstairs","session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
91012024-02-27T21:47:31.651Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010854871Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
91022024-02-27T21:47:31.651Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010877458Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
91032024-02-27T21:47:31.651Z{"msg":"bfba2fa9-5eb6-45b8-a083-f46659b623d5 is now active with session: 2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010905685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
91042024-02-27T21:47:31.651Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:30.010928472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480,"session_id":"2e47933a-7de6-4ae1-9d58-2d5faa9d4bfb"}
91052024-02-27T21:47:31.651Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
91062024-02-27T21:47:31.651Zcrucible","level":30,"time":"2024-02-27T21:47:30.010964588Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480}
91072024-02-27T21:47:31.651ZLoading write count information from file "/tmp/test_repair_verify.data"
91082024-02-27T21:47:31.651ZRun Repair workload
91092024-02-27T21:47:31.651Z01/30 Write block 26 len 40960 data: 8 7 6 7 9 9 8 8 11 9
91102024-02-27T21:47:31.651Z02/30 Flush
91112024-02-27T21:47:31.651Z03/30 Write block 351 len 24576 data: 8 8 6 6 9 8
91122024-02-27T21:47:31.651Z04/30 Write block 391 len 40960 data: 8 6 7 5 5 4 7 6 7 9
91132024-02-27T21:47:31.651Z05/30 Write block 337 len 4096 data: 8
91142024-02-27T21:47:31.651Z06/30 Write block 438 len 20480 data: 4 5 4 2 3
91152024-02-27T21:47:31.651Z07/30 Read block 479 len 32768
91162024-02-27T21:47:31.652Z08/30 Write block 212 len 16384 data: 10 8 8 7
91172024-02-27T21:47:31.652Z09/30 Read block 254 len 8192
91182024-02-27T21:47:31.652Z10/30 Read block 70 len 12288
91192024-02-27T21:47:31.652Z11/30 Read block 443 len 8192
91202024-02-27T21:47:31.652Z12/30 Write block 44 len 36864 data: 9 10 6 6 6 6 7 5 6
91212024-02-27T21:47:31.652Z13/30 Read block 577 len 40960
91222024-02-27T21:47:31.652Z14/30 Read block 102 len 8192
91232024-02-27T21:47:31.652Z15/30 Read block 248 len 36864
91242024-02-27T21:47:31.652Z16/30 Flush
91252024-02-27T21:47:31.652Z17/30 Flush
91262024-02-27T21:47:31.652Z18/30 Flush
91272024-02-27T21:47:31.652Z19/30 Flush
91282024-02-27T21:47:31.652Z20/30 Write block 579 len 16384 data: 6 6 5 6
91292024-02-27T21:47:31.652Z21/30 Read block 368 len 8192
91302024-02-27T21:47:31.653Z22/30 Write block 21 len 8192 data: 6 5
91312024-02-27T21:47:31.653Z23/30 Flush
91322024-02-27T21:47:31.653Z24/30 Write block 41 len 32768 data: 8 8 10 10 11 7 7 7
91332024-02-27T21:47:31.653Z25/30 Read block 59 len 32768
91342024-02-27T21:47:31.653Z26/30 Write block 353 len 20480 data: 7 7 10 9 5
91352024-02-27T21:47:31.658Z27/30 Write block 427 len 36864 data: 4 4 6 6 6 7 9 11 8
91362024-02-27T21:47:31.658Z28/30 Write block 209 len 12288 data: 9 9 9
91372024-02-27T21:47:31.658Z29/30 Read block 225 len 4096
91382024-02-27T21:47:31.658Z30/30 Write block 254 len 16384 data: 6 9 9 6
91392024-02-27T21:47:31.658Z----------------------------------------------------------------
91402024-02-27T21:47:31.658Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
91412024-02-27T21:47:31.659ZStates: Active Active Active
91422024-02-27T21:47:31.659ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91432024-02-27T21:47:31.659Z 1 Acked 1000 Write 10 Sent Done Done false
91442024-02-27T21:47:31.659Z 2 Acked 1001 Flush 0 Sent Done Done false
91452024-02-27T21:47:31.659Z 3 Acked 1002 Write 6 Sent Done Done false
91462024-02-27T21:47:31.659Z 4 Acked 1003 Write 10 Sent Done Done false
91472024-02-27T21:47:31.659Z 5 Acked 1004 Write 1 Sent Done Done false
91482024-02-27T21:47:31.659Z 6 Acked 1005 Write 5 Sent Done Done false
91492024-02-27T21:47:31.659Z 7 Acked 1006 Read 8 Sent Done Done false
91502024-02-27T21:47:31.659Z 8 Acked 1007 Write 4 Sent Done Done false
91512024-02-27T21:47:31.659Z 9 Acked 1008 Read 2 Sent Done Done false
91522024-02-27T21:47:31.659Z 10 Acked 1009 Read 3 Sent Done Done false
91532024-02-27T21:47:31.659Z 11 Acked 1010 Read 2 Sent Done Done false
91542024-02-27T21:47:31.659Z 12 Acked 1011 Write 9 Sent Done Done false
91552024-02-27T21:47:31.659Z 13 Acked 1012 Read 10 Sent Done Done false
91562024-02-27T21:47:31.660Z 14 Acked 1013 Read 2 Sent Done Done false
91572024-02-27T21:47:31.660Z 15 Acked 1014 Read 9 Sent Done Done false
91582024-02-27T21:47:31.660Z 16 Acked 1015 Flush 0 Sent Done Done false
91592024-02-27T21:47:31.660Z 17 Acked 1016 Flush 0 Sent Done Done false
91602024-02-27T21:47:31.660Z 18 Acked 1017 Flush 0 Sent Done Done false
91612024-02-27T21:47:31.660Z 19 Acked 1018 Flush 0 Sent Done Done false
91622024-02-27T21:47:31.660Z 20 Acked 1019 Write 4 Sent Done Done false
91632024-02-27T21:47:31.660Z 21 Acked 1020 Read 2 Sent Done Done false
91642024-02-27T21:47:31.660Z 22 Acked 1021 Write 2 Sent Done Done false
91652024-02-27T21:47:31.660Z 23 Acked 1022 Flush 0 Sent Done Done false
91662024-02-27T21:47:31.660Z 24 Acked 1023 Write 8 Sent Done Done false
91672024-02-27T21:47:31.660Z 25 Acked 1024 Read 8 Sent Done Done false
91682024-02-27T21:47:31.660Z 26 Acked 1025 Write 5 Sent Done Done false
91692024-02-27T21:47:31.660Z 27 Acked 1026 Write 9 Sent Done Done false
91702024-02-27T21:47:31.660Z 28 Acked 1027 Write 3 Sent Done Done false
91712024-02-27T21:47:31.660Z 29 Acked 1028 Read 1 Sent Done Done false
91722024-02-27T21:47:31.660Z 30 Acked 1029 Write 4 Sent Sent Sent false
91732024-02-27T21:47:31.660Z STATES DS:0 DS:1 DS:2 TOTAL
91742024-02-27T21:47:31.660Z New 0 0 0 0
91752024-02-27T21:47:31.660Z Sent 30 1 1 32
91762024-02-27T21:47:31.661Z Done 0 29 29 58
91772024-02-27T21:47:31.661Z Skipped 0 0 0 0
91782024-02-27T21:47:31.661Z Error 0 0 0 0
91792024-02-27T21:47:31.661ZLast Flush: 0 1022 1022
91802024-02-27T21:47:31.661ZDownstairs last five completed:
91812024-02-27T21:47:31.661ZUpstairs last five completed: 30 29 28 27 26
91822024-02-27T21:47:31.661Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:30.042393266Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1480}
91832024-02-27T21:47:31.661ZWrote out file "/tmp/test_repair_verify.data"
91842024-02-27T21:47:31.667Z
91852024-02-27T21:47:31.667ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91862024-02-27T21:47:31.679ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91872024-02-27T21:47:31.679Z 0 000-019 83 83 83 153 153 153 F F F
91882024-02-27T21:47:31.679Z 1 020-039 82 84 84 152 159 159 F F F <---
91892024-02-27T21:47:31.679Z 2 040-059 76 84 84 144 155 155 F T T <---
91902024-02-27T21:47:31.679Z 3 060-079 78 78 78 145 145 145 F F F
91912024-02-27T21:47:31.679Z 4 080-099 82 82 82 151 151 151 F F F
91922024-02-27T21:47:31.679Z 5 100-119 78 78 78 145 145 145 F F F
91932024-02-27T21:47:31.680Z 6 120-139 76 76 76 144 144 144 F F F
91942024-02-27T21:47:31.680Z 7 140-159 78 78 78 145 145 145 F F F
91952024-02-27T21:47:31.680Z 8 160-179 81 81 81 149 149 149 F F F
91962024-02-27T21:47:31.680Z 9 180-199 82 82 82 151 151 151 F F F
91972024-02-27T21:47:31.680Z 10 200-219 71 84 84 134 155 155 F T T <---
91982024-02-27T21:47:31.680Z 11 220-239 82 82 82 150 150 150 F F F
91992024-02-27T21:47:31.680Z 12 240-259 81 81 81 149 149 149 F T T <---
92002024-02-27T21:47:31.685Z 13 260-279 81 81 81 149 149 149 F F F
92012024-02-27T21:47:31.685Z 14 280-299 74 74 74 141 141 141 F F F
92022024-02-27T21:47:31.685Z 15 300-319 82 82 82 150 150 150 F F F
92032024-02-27T21:47:31.685Z 16 320-339 82 84 84 150 155 155 F F F <---
92042024-02-27T21:47:31.685Z 17 340-359 83 84 84 153 155 155 F T T <---
92052024-02-27T21:47:31.685Z 18 360-379 82 82 82 151 151 151 F F F
92062024-02-27T21:47:31.685Z 19 380-399 83 84 84 153 155 155 F F F <---
92072024-02-27T21:47:31.685Z 20 400-419 79 84 84 146 155 155 F F F <---
92082024-02-27T21:47:31.685Z 21 420-439 80 84 84 148 155 155 F T T <---
92092024-02-27T21:47:31.686Z 22 440-459 74 84 84 141 155 155 F F F <---
92102024-02-27T21:47:31.686Z 23 460-479 79 79 79 146 146 146 F F F
92112024-02-27T21:47:31.686Z 24 480-499 82 82 82 151 151 151 F F F
92122024-02-27T21:47:31.686Z 25 500-519 76 76 76 144 144 144 F F F
92132024-02-27T21:47:31.686Z 26 520-539 83 83 83 153 153 153 F F F
92142024-02-27T21:47:31.686Z 27 540-559 83 83 83 153 153 153 F F F
92152024-02-27T21:47:31.686Z 28 560-579 83 84 84 153 159 159 F F F <---
92162024-02-27T21:47:31.686Z 29 580-599 71 84 84 134 159 159 F F F <---
92172024-02-27T21:47:31.686ZMax gen: 84, Max flush: 159
92182024-02-27T21:47:31.686ZError: Difference in extent metadata found!
92192024-02-27T21:47:31.686ZA Difference in extent metadata is expected here
92202024-02-27T21:47:31.686ZOn loop 42
92212024-02-27T21:47:31.686Z
92222024-02-27T21:47:31.690ZVerifying data now
92232024-02-27T21:47:36.312ZNo change after verify
92242024-02-27T21:47:36.313ZLoop: 42 Downstairs dump after verify (and repair):
92252024-02-27T21:47:36.326ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92262024-02-27T21:47:36.327Z 0 000-019 83 83 83 153 153 153 F F F
92272024-02-27T21:47:36.327Z 1 020-039 84 84 84 159 159 159 F F F
92282024-02-27T21:47:36.327Z 2 040-059 85 85 85 160 160 160 F F F
92292024-02-27T21:47:36.327Z 3 060-079 78 78 78 145 145 145 F F F
92302024-02-27T21:47:36.327Z 4 080-099 82 82 82 151 151 151 F F F
92312024-02-27T21:47:36.327Z 5 100-119 78 78 78 145 145 145 F F F
92322024-02-27T21:47:36.332Z 6 120-139 76 76 76 144 144 144 F F F
92332024-02-27T21:47:36.332Z 7 140-159 78 78 78 145 145 145 F F F
92342024-02-27T21:47:36.332Z 8 160-179 81 81 81 149 149 149 F F F
92352024-02-27T21:47:36.333Z 9 180-199 82 82 82 151 151 151 F F F
92362024-02-27T21:47:36.333Z 10 200-219 85 85 85 160 160 160 F F F
92372024-02-27T21:47:36.333Z 11 220-239 82 82 82 150 150 150 F F F
92382024-02-27T21:47:36.333Z 12 240-259 85 85 85 160 160 160 F F F
92392024-02-27T21:47:36.333Z 13 260-279 81 81 81 149 149 149 F F F
92402024-02-27T21:47:36.333Z 14 280-299 74 74 74 141 141 141 F F F
92412024-02-27T21:47:36.334Z 15 300-319 82 82 82 150 150 150 F F F
92422024-02-27T21:47:36.334Z 16 320-339 84 84 84 155 155 155 F F F
92432024-02-27T21:47:36.334Z 17 340-359 85 85 85 160 160 160 F F F
92442024-02-27T21:47:36.334Z 18 360-379 82 82 82 151 151 151 F F F
92452024-02-27T21:47:36.334Z 19 380-399 84 84 84 155 155 155 F F F
92462024-02-27T21:47:36.334Z 20 400-419 84 84 84 155 155 155 F F F
92472024-02-27T21:47:36.335Z 21 420-439 85 85 85 160 160 160 F F F
92482024-02-27T21:47:36.335Z 22 440-459 84 84 84 155 155 155 F F F
92492024-02-27T21:47:36.335Z 23 460-479 79 79 79 146 146 146 F F F
92502024-02-27T21:47:36.335Z 24 480-499 82 82 82 151 151 151 F F F
92512024-02-27T21:47:36.335Z 25 500-519 76 76 76 144 144 144 F F F
92522024-02-27T21:47:36.335Z 26 520-539 83 83 83 153 153 153 F F F
92532024-02-27T21:47:36.335Z 27 540-559 83 83 83 153 153 153 F F F
92542024-02-27T21:47:36.335Z 28 560-579 84 84 84 159 159 159 F F F
92552024-02-27T21:47:36.336Z 29 580-599 84 84 84 159 159 159 F F F
92562024-02-27T21:47:36.336ZMax gen: 85, Max flush: 160
92572024-02-27T21:47:36.336Z
92582024-02-27T21:47:36.336ZBegin loop 43
92592024-02-27T21:47:36.337ZDownstairs to restart: 0
92602024-02-27T21:47:36.347Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.735966249Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488}
92612024-02-27T21:47:36.348Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.73621475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488}
92622024-02-27T21:47:36.348Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736243787Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488}
92632024-02-27T21:47:36.348Z{"msg":"Crucible 07488cd4-4ce2-4335-8826-8d7567f0518d has session id: 9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
92642024-02-27T21:47:36.348Z2024-02-27T21:47:34.736269414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92652024-02-27T21:47:36.356Z{"msg":"Upstairs opts: Upstairs UUID: 07488cd4-4ce2-4335-8826-8d7567f0518d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736314768Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92662024-02-27T21:47:36.356Z{"msg":"Crucible stats registered with UUID: 07488cd4-4ce2-4335-8826-8d7567f0518d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736338736Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92672024-02-27T21:47:36.356Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736360413Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488}
92682024-02-27T21:47:36.357Z{"msg":"07488cd4-4ce2-4335-8826-8d7567f0518d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736528793Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92692024-02-27T21:47:36.357Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736567009Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92702024-02-27T21:47:36.357Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736596835Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92712024-02-27T21:47:36.357Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736626792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92722024-02-27T21:47:36.357Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736666637Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92732024-02-27T21:47:36.357Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736685295Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92742024-02-27T21:47:36.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736717591Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92752024-02-27T21:47:36.358Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736739378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92762024-02-27T21:47:36.358Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736766365Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92772024-02-27T21:47:36.358Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736804181Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92782024-02-27T21:47:36.358Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.736969171Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92792024-02-27T21:47:36.358Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.737025785Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92802024-02-27T21:47:36.358Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.737053561Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92812024-02-27T21:47:36.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.737607596Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92822024-02-27T21:47:36.358Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.737647802Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92832024-02-27T21:47:36.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.737674218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92842024-02-27T21:47:36.358Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.737703635Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92852024-02-27T21:47:36.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.737726352Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92862024-02-27T21:47:36.358Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.73774629Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92872024-02-27T21:47:36.358Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738245521Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92882024-02-27T21:47:36.358Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738292626Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92892024-02-27T21:47:36.358Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738312173Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92902024-02-27T21:47:36.358Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738778768Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92912024-02-27T21:47:36.359Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738819513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92922024-02-27T21:47:36.359Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.73884638Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92932024-02-27T21:47:36.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738884756Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92942024-02-27T21:47:36.359Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738926251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92952024-02-27T21:47:36.359Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738958507Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92962024-02-27T21:47:36.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738984814Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92972024-02-27T21:47:36.359Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.738999862Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92982024-02-27T21:47:36.359Z{"msg":"[0]R flush_numbers[0..12]: [153, 159, 160, 145, 151, 145, 144, 145, 149, 151, 160, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.73901571Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
92992024-02-27T21:47:36.359Z{"msg":"[0]R generation[0..12]: [83, 84, 85, 78, 82, 78, 76, 78, 81, 82, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739042067Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93002024-02-27T21:47:36.359Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739057185Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93012024-02-27T21:47:36.359ZWait for a query_work_queue command to finish before sending IO
93022024-02-27T21:47:36.359Z{"msg":"[1]R flush_numbers[0..12]: [153, 159, 160, 145, 151, 145, 144, 145, 149, 151, 160, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739084312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93032024-02-27T21:47:36.359Z{"msg":"[1]R generation[0..12]: [83, 84, 85, 78, 82, 78, 76, 78, 81, 82, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.73910757Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93042024-02-27T21:47:36.359Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739139046Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93052024-02-27T21:47:36.359Z{"msg":"[2]R flush_numbers[0..12]: [153, 159, 160, 145, 151, 145, 144, 145, 149, 151, 160, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739155834Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93062024-02-27T21:47:36.359Z{"msg":"[2]R generation[0..12]: [83, 84, 85, 78, 82, 78, 76, 78, 81, 82, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739177731Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93072024-02-27T21:47:36.359Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739193179Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93082024-02-27T21:47:36.359Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739215007Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93092024-02-27T21:47:36.359Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739228955Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93102024-02-27T21:47:36.359Z{"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739248963Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93112024-02-27T21:47:36.359Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739262681Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"":"downstairs","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93122024-02-27T21:47:36.360Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739283079Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93132024-02-27T21:47:36.360Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739295817Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"0","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93142024-02-27T21:47:36.360Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739315555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"1","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93152024-02-27T21:47:36.360Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739329693Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"client":"2","":"downstairs","session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93162024-02-27T21:47:36.360Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739350431Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93172024-02-27T21:47:36.360Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739363249Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93182024-02-27T21:47:36.360Z{"msg":"07488cd4-4ce2-4335-8826-8d7567f0518d is now active with session: 9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739377068Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93192024-02-27T21:47:36.360Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739389186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488,"session_id":"9a43a57f-5d8b-4d65-b74f-ab3ad510a9ff"}
93202024-02-27T21:47:36.360Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:34.739401555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488}
93212024-02-27T21:47:36.360ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93222024-02-27T21:47:36.360ZLoading write count information from file "/tmp/test_repair_verify.data"
93232024-02-27T21:47:36.360ZRun Repair workload
93242024-02-27T21:47:36.360Z01/30 Write block 3 len 8192 data: 2 4
93252024-02-27T21:47:36.364Z02/30 Read block 53 len 36864
93262024-02-27T21:47:36.364Z03/30 Write block 434 len 40960 data: 12 9 6 6 5 6 5 3 4 5
93272024-02-27T21:47:36.364Z04/30 Write block 578 len 4096 data: 8
93282024-02-27T21:47:36.365Z05/30 Read block 578 len 28672
93292024-02-27T21:47:36.365Z06/30 Flush
93302024-02-27T21:47:36.365Z07/30 Flush
93312024-02-27T21:47:36.365Z08/30 Write block 105 len 12288 data: 6 7 10
93322024-02-27T21:47:36.365Z09/30 Read block 99 len 20480
93332024-02-27T21:47:36.365Z10/30 Read block 195 len 40960
93342024-02-27T21:47:36.366Z11/30 Read block 160 len 32768
93352024-02-27T21:47:36.366Z12/30 Write block 189 len 24576 data: 9 8 8 8 6 6
93362024-02-27T21:47:36.366Z13/30 Read block 52 len 20480
93372024-02-27T21:47:36.369Z14/30 Flush
93382024-02-27T21:47:36.370Z15/30 Flush
93392024-02-27T21:47:36.370Z16/30 Write block 129 len 36864 data: 5 5 4 2 3 3 5 5 5
93402024-02-27T21:47:36.370Z17/30 Read block 530 len 32768
93412024-02-27T21:47:36.374Z18/30 Write block 542 len 28672 data: 6 5 3 3 4 4 9
93422024-02-27T21:47:36.374Z19/30 Read block 222 len 20480
93432024-02-27T21:47:36.374Z20/30 Read block 505 len 36864
93442024-02-27T21:47:36.378Z21/30 Write block 425 len 28672 data: 4 4 5 5 7 7 7
93452024-02-27T21:47:36.378Z22/30 Write block 51 len 8192 data: 6 7
93462024-02-27T21:47:36.378Z23/30 Read block 425 len 24576
93472024-02-27T21:47:36.378Z24/30 Flush
93482024-02-27T21:47:36.383Z25/30 Write block 15 len 24576 data: 10 10 9 7 6 6
93492024-02-27T21:47:36.383Z26/30 Read block 551 len 16384
93502024-02-27T21:47:36.383Z27/30 Write block 101 len 28672 data: 7 7 6 5 7 8 11
93512024-02-27T21:47:36.383Z28/30 Write block 469 len 12288 data: 8 6 6
93522024-02-27T21:47:36.383Z29/30 Write block 62 len 8192 data: 5 7
93532024-02-27T21:47:36.383Z30/30 Write block 340 len 8192 data: 9 9
93542024-02-27T21:47:36.389Z----------------------------------------------------------------
93552024-02-27T21:47:36.389Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
93562024-02-27T21:47:36.389ZStates: Active Active Active
93572024-02-27T21:47:36.389ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93582024-02-27T21:47:36.389Z 1 Acked 1000 Write 2 Done Done Done false
93592024-02-27T21:47:36.389Z 2 Acked 1001 Read 9 Sent Done Done false
93602024-02-27T21:47:36.389Z 3 Acked 1002 Write 10 Sent Done Done false
93612024-02-27T21:47:36.390Z 4 Acked 1003 Write 1 Sent Done Done false
93622024-02-27T21:47:36.390Z 5 Acked 1004 Read 7 Sent Done Done false
93632024-02-27T21:47:36.390Z 6 Acked 1005 Flush 0 Sent Done Done false
93642024-02-27T21:47:36.390Z 7 Acked 1006 Flush 0 Sent Done Done false
93652024-02-27T21:47:36.390Z 8 Acked 1007 Write 3 Sent Done Done false
93662024-02-27T21:47:36.390Z 9 Acked 1008 Read 5 Sent Done Done false
93672024-02-27T21:47:36.390Z 10 Acked 1009 Read 10 Sent Done Done false
93682024-02-27T21:47:36.390Z 11 Acked 1010 Read 8 Sent Done Done false
93692024-02-27T21:47:36.390Z 12 Acked 1011 Write 6 Sent Done Done false
93702024-02-27T21:47:36.390Z 13 Acked 1012 Read 5 Sent Done Done false
93712024-02-27T21:47:36.390Z 14 Acked 1013 Flush 0 Sent Done Done false
93722024-02-27T21:47:36.390Z 15 Acked 1014 Flush 0 Sent Done Done false
93732024-02-27T21:47:36.390Z 16 Acked 1015 Write 9 Sent Done Done false
93742024-02-27T21:47:36.391Z 17 Acked 1016 Read 8 Sent Done Done false
93752024-02-27T21:47:36.391Z 18 Acked 1017 Write 7 Sent Done Done false
93762024-02-27T21:47:36.391Z 19 Acked 1018 Read 5 Sent Done Done false
93772024-02-27T21:47:36.391Z 20 Acked 1019 Read 9 Sent Done Done false
93782024-02-27T21:47:36.391Z 21 Acked 1020 Write 7 Sent Done Done false
93792024-02-27T21:47:36.391Z 22 Acked 1021 Write 2 Sent Done Done false
93802024-02-27T21:47:36.391Z 23 Acked 1022 Read 6 Sent Done Done false
93812024-02-27T21:47:36.391Z 24 Acked 1023 Flush 0 Sent Done Done false
93822024-02-27T21:47:36.391Z 25 Acked 1024 Write 6 Sent Done Done false
93832024-02-27T21:47:36.391Z 26 Acked 1025 Read 4 Sent Done Done false
93842024-02-27T21:47:36.391Z 27 Acked 1026 Write 7 Sent Done Done false
93852024-02-27T21:47:36.392Z 28 Acked 1027 Write 3 Sent Done Done false
93862024-02-27T21:47:36.392Z 29 Acked 1028 Write 2 Sent Done Done false
93872024-02-27T21:47:36.392Z 30 Acked 1029 Write 2 Sent Sent Sent false
93882024-02-27T21:47:36.392Z STATES DS:0 DS:1 DS:2 TOTAL
93892024-02-27T21:47:36.392Z New 0 0 0 0
93902024-02-27T21:47:36.392Z Sent 29 1 1 31
93912024-02-27T21:47:36.392Z Done 1 29 29 59
93922024-02-27T21:47:36.392Z Skipped 0 0 0 0
93932024-02-27T21:47:36.392Z Error 0 0 0 0
93942024-02-27T21:47:36.392ZLast Flush: 0 1023 1023
93952024-02-27T21:47:36.392ZDownstairs last five completed:
93962024-02-27T21:47:36.392ZUpstairs last five completed: 30 29 28 27 26
93972024-02-27T21:47:36.393Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:34.771856472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1488}
93982024-02-27T21:47:36.393ZWrote out file "/tmp/test_repair_verify.data"
93992024-02-27T21:47:36.393Z
94002024-02-27T21:47:36.397ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
94012024-02-27T21:47:36.409ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94022024-02-27T21:47:36.409Z 0 000-019 83 86 86 153 161 161 T T T <---
94032024-02-27T21:47:36.409Z 1 020-039 84 84 84 159 159 159 F T T <---
94042024-02-27T21:47:36.409Z 2 040-059 85 86 86 160 165 165 F F F <---
94052024-02-27T21:47:36.409Z 3 060-079 78 78 78 145 145 145 F T T <---
94062024-02-27T21:47:36.409Z 4 080-099 82 82 82 151 151 151 F F F
94072024-02-27T21:47:36.409Z 5 100-119 78 86 86 145 163 163 F T T <---
94082024-02-27T21:47:36.409Z 6 120-139 76 86 86 144 165 165 F F F <---
94092024-02-27T21:47:36.410Z 7 140-159 78 78 78 145 145 145 F F F
94102024-02-27T21:47:36.410Z 8 160-179 81 81 81 149 149 149 F F F
94112024-02-27T21:47:36.410Z 9 180-199 82 86 86 151 163 163 F F F <---
94122024-02-27T21:47:36.410Z 10 200-219 85 85 85 160 160 160 F F F
94132024-02-27T21:47:36.410Z 11 220-239 82 82 82 150 150 150 F F F
94142024-02-27T21:47:36.410Z 12 240-259 85 85 85 160 160 160 F F F
94152024-02-27T21:47:36.410Z 13 260-279 81 81 81 149 149 149 F F F
94162024-02-27T21:47:36.410Z 14 280-299 74 74 74 141 141 141 F F F
94172024-02-27T21:47:36.411Z 15 300-319 82 82 82 150 150 150 F F F
94182024-02-27T21:47:36.411Z 16 320-339 84 84 84 155 155 155 F F F
94192024-02-27T21:47:36.411Z 17 340-359 85 85 85 160 160 160 F T T <---
94202024-02-27T21:47:36.411Z 18 360-379 82 82 82 151 151 151 F F F
94212024-02-27T21:47:36.411Z 19 380-399 84 84 84 155 155 155 F F F
94222024-02-27T21:47:36.411Z 20 400-419 84 84 84 155 155 155 F F F
94232024-02-27T21:47:36.411Z 21 420-439 85 86 86 160 165 165 F F F <---
94242024-02-27T21:47:36.411Z 22 440-459 84 86 86 155 161 161 F F F <---
94252024-02-27T21:47:36.411Z 23 460-479 79 79 79 146 146 146 F T T <---
94262024-02-27T21:47:36.411Z 24 480-499 82 82 82 151 151 151 F F F
94272024-02-27T21:47:36.411Z 25 500-519 76 76 76 144 144 144 F F F
94282024-02-27T21:47:36.412Z 26 520-539 83 83 83 153 153 153 F F F
94292024-02-27T21:47:36.412Z 27 540-559 83 86 86 153 165 165 F F F <---
94302024-02-27T21:47:36.412Z 28 560-579 84 86 86 159 161 161 F F F <---
94312024-02-27T21:47:36.412Z 29 580-599 84 84 84 159 159 159 F F F
94322024-02-27T21:47:36.412ZMax gen: 86, Max flush: 165
94332024-02-27T21:47:36.419ZError: Difference in extent metadata found!
94342024-02-27T21:47:36.419ZA Difference in extent metadata is expected here
94352024-02-27T21:47:36.419ZOn loop 43
94362024-02-27T21:47:36.419Z
94372024-02-27T21:47:36.419ZVerifying data now
94382024-02-27T21:47:41.067ZNo change after verify
94392024-02-27T21:47:41.067ZLoop: 43 Downstairs dump after verify (and repair):
94402024-02-27T21:47:41.082ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94412024-02-27T21:47:41.082Z 0 000-019 87 87 87 166 166 166 F F F
94422024-02-27T21:47:41.082Z 1 020-039 87 87 87 166 166 166 F F F
94432024-02-27T21:47:41.082Z 2 040-059 86 86 86 165 165 165 F F F
94442024-02-27T21:47:41.083Z 3 060-079 87 87 87 166 166 166 F F F
94452024-02-27T21:47:41.083Z 4 080-099 82 82 82 151 151 151 F F F
94462024-02-27T21:47:41.087Z 5 100-119 87 87 87 166 166 166 F F F
94472024-02-27T21:47:41.087Z 6 120-139 86 86 86 165 165 165 F F F
94482024-02-27T21:47:41.087Z 7 140-159 78 78 78 145 145 145 F F F
94492024-02-27T21:47:41.087Z 8 160-179 81 81 81 149 149 149 F F F
94502024-02-27T21:47:41.087Z 9 180-199 86 86 86 163 163 163 F F F
94512024-02-27T21:47:41.087Z 10 200-219 85 85 85 160 160 160 F F F
94522024-02-27T21:47:41.087Z 11 220-239 82 82 82 150 150 150 F F F
94532024-02-27T21:47:41.088Z 12 240-259 85 85 85 160 160 160 F F F
94542024-02-27T21:47:41.088Z 13 260-279 81 81 81 149 149 149 F F F
94552024-02-27T21:47:41.088Z 14 280-299 74 74 74 141 141 141 F F F
94562024-02-27T21:47:41.088Z 15 300-319 82 82 82 150 150 150 F F F
94572024-02-27T21:47:41.088Z 16 320-339 84 84 84 155 155 155 F F F
94582024-02-27T21:47:41.088Z 17 340-359 87 87 87 166 166 166 F F F
94592024-02-27T21:47:41.088Z 18 360-379 82 82 82 151 151 151 F F F
94602024-02-27T21:47:41.088Z 19 380-399 84 84 84 155 155 155 F F F
94612024-02-27T21:47:41.088Z 20 400-419 84 84 84 155 155 155 F F F
94622024-02-27T21:47:41.088Z 21 420-439 86 86 86 165 165 165 F F F
94632024-02-27T21:47:41.088Z 22 440-459 86 86 86 161 161 161 F F F
94642024-02-27T21:47:41.088Z 23 460-479 87 87 87 166 166 166 F F F
94652024-02-27T21:47:41.088Z 24 480-499 82 82 82 151 151 151 F F F
94662024-02-27T21:47:41.089Z 25 500-519 76 76 76 144 144 144 F F F
94672024-02-27T21:47:41.089Z 26 520-539 83 83 83 153 153 153 F F F
94682024-02-27T21:47:41.089Z 27 540-559 86 86 86 165 165 165 F F F
94692024-02-27T21:47:41.089Z 28 560-579 86 86 86 161 161 161 F F F
94702024-02-27T21:47:41.089Z 29 580-599 84 84 84 159 159 159 F F F
94712024-02-27T21:47:41.089ZMax gen: 87, Max flush: 166
94722024-02-27T21:47:41.089Z
94732024-02-27T21:47:41.089ZBegin loop 44
94742024-02-27T21:47:41.089ZDownstairs to restart: 1
94752024-02-27T21:47:41.105Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.492505027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496}
94762024-02-27T21:47:41.105Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.492770666Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496}
94772024-02-27T21:47:41.105Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.492792433Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496}
94782024-02-27T21:47:41.112Z{"msg":"Crucible 45338ebb-bc86-47a5-8b3b-4f54c55e0eeb has session id: 5e739909-6f55-4e85-924a-8a7af8fe1f58","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.492832619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94792024-02-27T21:47:41.112Z{"msg":"Upstairs opts: Upstairs UUID: 45338ebb-bc86-47a5-8b3b-4f54c55e0eeb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
94802024-02-27T21:47:41.113Z,"time":"2024-02-27T21:47:39.492873044Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94812024-02-27T21:47:41.113Z{"msg":"Crucible stats registered with UUID: 45338ebb-bc86-47a5-8b3b-4f54c55e0eeb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.492914369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94822024-02-27T21:47:41.113Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.492987501Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496}
94832024-02-27T21:47:41.113Z{"msg":"45338ebb-bc86-47a5-8b3b-4f54c55e0eeb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493106716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94842024-02-27T21:47:41.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493134463Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94852024-02-27T21:47:41.113Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493154941Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94862024-02-27T21:47:41.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493192126Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94872024-02-27T21:47:41.113Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493221123Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94882024-02-27T21:47:41.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.4932468Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94892024-02-27T21:47:41.113Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493261378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94902024-02-27T21:47:41.113Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493280786Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94912024-02-27T21:47:41.113Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493317451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94922024-02-27T21:47:41.113Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493376825Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94932024-02-27T21:47:41.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493492151Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94942024-02-27T21:47:41.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493551214Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94952024-02-27T21:47:41.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.493598958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94962024-02-27T21:47:41.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.494213126Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94972024-02-27T21:47:41.114Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.494259591Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94982024-02-27T21:47:41.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.494287217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
94992024-02-27T21:47:41.114Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.494315834Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95002024-02-27T21:47:41.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.494342451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95012024-02-27T21:47:41.114Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.494369488Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95022024-02-27T21:47:41.114Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.494887717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95032024-02-27T21:47:41.114Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.494930872Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95042024-02-27T21:47:41.114Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.494993934Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95052024-02-27T21:47:41.114Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495492026Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95062024-02-27T21:47:41.115Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495531701Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95072024-02-27T21:47:41.115Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495547429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95082024-02-27T21:47:41.115Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495569206Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95092024-02-27T21:47:41.115Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495584315Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95102024-02-27T21:47:41.115Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495598503Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95112024-02-27T21:47:41.115Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495617701Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95122024-02-27T21:47:41.115Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495631759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95132024-02-27T21:47:41.115Z{"msg":"[0]R flush_numbers[0..12]: [166, 166, 165, 166, 151, 166, 165, 145, 149, 163, 160, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495652836Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95142024-02-27T21:47:41.115Z{"msg":"[0]R generation[0..12]: [87, 87, 86, 87, 82, 87, 86, 78, 81, 86, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495667225Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95152024-02-27T21:47:41.115Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
95162024-02-27T21:47:41.115Z,"time":"2024-02-27T21:47:39.495692252Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95172024-02-27T21:47:41.115Z{"msg":"[1]R flush_numbers[0..12]: [166, 166, 165, 166, 151, 166, 165, 145, 149, 163, 160, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495713159Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95182024-02-27T21:47:41.115Z{"msg":"[1]R generation[0..12]: [87, 87, 86, 87, 82, 87, 86, 78, 81, 86, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495732127Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95192024-02-27T21:47:41.116Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495755794Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95202024-02-27T21:47:41.116Z{"msg":"[2]R flush_numbers[0..12]: [166, 166, 165, 166, 151, 166, 165, 145, 149, 163, 160, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495770853Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95212024-02-27T21:47:41.116Z{"msg":"[2]R generation[0..12]: [87, 87, 86, 87, 82, 87, 86, 78, 81, 86, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495785141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95222024-02-27T21:47:41.116Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495799099Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95232024-02-27T21:47:41.116Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495819747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95242024-02-27T21:47:41.116Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495833945Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95252024-02-27T21:47:41.116Z{"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495847484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95262024-02-27T21:47:41.116Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495860772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"":"downstairs","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95272024-02-27T21:47:41.116Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.49588184Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95282024-02-27T21:47:41.116Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495894718Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"0","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95292024-02-27T21:47:41.116Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495908676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"1","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95302024-02-27T21:47:41.116Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495938613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"client":"2","":"downstairs","session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95312024-02-27T21:47:41.116Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495954101Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95322024-02-27T21:47:41.116Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495967139Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95332024-02-27T21:47:41.117Z{"msg":"45338ebb-bc86-47a5-8b3b-4f54c55e0eeb is now active with session: 5e739909-6f55-4e85-924a-8a7af8fe1f58","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.495980358Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95342024-02-27T21:47:41.117Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.496013724Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496,"session_id":"5e739909-6f55-4e85-924a-8a7af8fe1f58"}
95352024-02-27T21:47:41.117Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:39.496028262Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496}
95362024-02-27T21:47:41.117ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95372024-02-27T21:47:41.117ZLoading write count information from file "/tmp/test_repair_verify.data"
95382024-02-27T21:47:41.117ZRun Repair workload
95392024-02-27T21:47:41.117Z01/30 Write block 414 len 8192 data: 5 6
95402024-02-27T21:47:41.122Z02/30 Write block 295 len 8192 data: 9 8
95412024-02-27T21:47:41.122Z03/30 Read block 286 len 4096
95422024-02-27T21:47:41.122Z04/30 Write block 417 len 8192 data: 9 8
95432024-02-27T21:47:41.122Z05/30 Write block 416 len 12288 data: 7 10 9
95442024-02-27T21:47:41.122Z06/30 Write block 170 len 8192 data: 5 4
95452024-02-27T21:47:41.123Z07/30 Read block 31 len 24576
95462024-02-27T21:47:41.123Z08/30 Read block 572 len 16384
95472024-02-27T21:47:41.123Z09/30 Flush
95482024-02-27T21:47:41.123Z10/30 Write block 486 len 24576 data: 12 11 11 13 11 11
95492024-02-27T21:47:41.123Z11/30 Read block 298 len 28672
95502024-02-27T21:47:41.123Z12/30 Read block 534 len 12288
95512024-02-27T21:47:41.123Z13/30 Read block 214 len 8192
95522024-02-27T21:47:41.123Z14/30 Read block 196 len 28672
95532024-02-27T21:47:41.123Z15/30 Write block 544 len 32768 data: 4 4 5 5 10 10 9 12
95542024-02-27T21:47:41.124Z16/30 Read block 121 len 16384
95552024-02-27T21:47:41.130Z17/30 Read block 490 len 16384
95562024-02-27T21:47:41.130Z18/30 Write block 338 len 28672 data: 9 9 10 10 9 8 8
95572024-02-27T21:47:41.130Z19/30 Write block 440 len 8192 data: 6 4
95582024-02-27T21:47:41.130Z20/30 Flush
95592024-02-27T21:47:41.130Z21/30 Flush
95602024-02-27T21:47:41.130Z22/30 Write block 331 len 4096 data: 7
95612024-02-27T21:47:41.142Z23/30 Flush
95622024-02-27T21:47:41.142Z24/30 Write block 219 len 36864 data: 9 8 8 9 8 9 6 8 8
95632024-02-27T21:47:41.142Z25/30 Read block 379 len 8192
95642024-02-27T21:47:41.142Z26/30 Write block 1 len 32768 data: 2 3 3 5 5 7 7 7
95652024-02-27T21:47:41.142Z27/30 Read block 356 len 12288
95662024-02-27T21:47:41.142Z28/30 Read block 381 len 24576
95672024-02-27T21:47:41.142Z29/30 Read block 287 len 4096
95682024-02-27T21:47:41.142Z30/30 Read block 352 len 28672
95692024-02-27T21:47:41.143Z----------------------------------------------------------------
95702024-02-27T21:47:41.143Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95712024-02-27T21:47:41.143ZStates: Active Active Active
95722024-02-27T21:47:41.143ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95732024-02-27T21:47:41.143Z 1 Acked 1000 Write 2 Done Done Done false
95742024-02-27T21:47:41.143Z 2 Acked 1001 Write 2 Done Done Done false
95752024-02-27T21:47:41.143Z 3 Acked 1002 Read 1 Done Done Done false
95762024-02-27T21:47:41.144Z 4 Acked 1003 Write 2 Done Sent Done false
95772024-02-27T21:47:41.144Z 5 Acked 1004 Write 3 Done Sent Done false
95782024-02-27T21:47:41.144Z 6 Acked 1005 Write 2 Done Sent Done false
95792024-02-27T21:47:41.144Z 7 Acked 1006 Read 6 Done Sent Done false
95802024-02-27T21:47:41.144Z 8 Acked 1007 Read 4 Done Sent Done false
95812024-02-27T21:47:41.144Z 9 Acked 1008 Flush 0 Done Sent Done false
95822024-02-27T21:47:41.144Z 10 Acked 1009 Write 6 Done Sent Done false
95832024-02-27T21:47:41.144Z 11 Acked 1010 Read 7 Done Sent Done false
95842024-02-27T21:47:41.145Z 12 Acked 1011 Read 3 Done Sent Done false
95852024-02-27T21:47:41.145Z 13 Acked 1012 Read 2 Done Sent Done false
95862024-02-27T21:47:41.145Z 14 Acked 1013 Read 7 Done Sent Done false
95872024-02-27T21:47:41.145Z 15 Acked 1014 Write 8 Done Sent Done false
95882024-02-27T21:47:41.145Z 16 Acked 1015 Read 4 Done Sent Done false
95892024-02-27T21:47:41.145Z 17 Acked 1016 Read 4 Done Sent Done false
95902024-02-27T21:47:41.145Z 18 Acked 1017 Write 7 Done Sent Done false
95912024-02-27T21:47:41.145Z 19 Acked 1018 Write 2 Done Sent Done false
95922024-02-27T21:47:41.145Z 20 Acked 1019 Flush 0 Done Sent Done false
95932024-02-27T21:47:41.145Z 21 Acked 1020 Flush 0 Done Sent Done false
95942024-02-27T21:47:41.146Z 22 Acked 1021 Write 1 Done Sent Done false
95952024-02-27T21:47:41.146Z 23 Acked 1022 Flush 0 Done Sent Done false
95962024-02-27T21:47:41.146Z 24 Acked 1023 Write 9 Done Sent Done false
95972024-02-27T21:47:41.146Z 25 Acked 1024 Read 2 Done Sent Done false
95982024-02-27T21:47:41.146Z 26 Acked 1025 Write 8 Done Sent Done false
95992024-02-27T21:47:41.146Z 27 Acked 1026 Read 3 Done Sent Done false
96002024-02-27T21:47:41.146Z 28 Acked 1027 Read 6 Done Sent Done false
96012024-02-27T21:47:41.146Z 29 Acked 1028 Read 1 Done Sent Done false
96022024-02-27T21:47:41.146Z 30 Acked 1029 Read 7 Done Sent Done false
96032024-02-27T21:47:41.146Z STATES DS:0 DS:1 DS:2 TOTAL
96042024-02-27T21:47:41.146Z New 0 0 0 0
96052024-02-27T21:47:41.146Z Sent 0 27 0 27
96062024-02-27T21:47:41.147Z Done 30 3 30 63
96072024-02-27T21:47:41.147Z Skipped 0 0 0 0
96082024-02-27T21:47:41.147Z Error 0 0 0 0
96092024-02-27T21:47:41.147ZLast Flush: 1022 0 1022
96102024-02-27T21:47:41.147ZDownstairs last five completed:
96112024-02-27T21:47:41.147ZUpstairs last five completed: 30 29 28 27 26
96122024-02-27T21:47:41.147Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:39.525036075Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1496}
96132024-02-27T21:47:41.147ZWrote out file "/tmp/test_repair_verify.data"
96142024-02-27T21:47:41.152Z
96152024-02-27T21:47:41.152ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96162024-02-27T21:47:41.205ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96172024-02-27T21:47:41.205Z 0 000-019 87 87 87 166 166 166 T F T <---
96182024-02-27T21:47:41.205Z 1 020-039 87 87 87 166 166 166 F F F
96192024-02-27T21:47:41.205Z 2 040-059 86 86 86 165 165 165 F F F
96202024-02-27T21:47:41.206Z 3 060-079 87 87 87 166 166 166 F F F
96212024-02-27T21:47:41.206Z 4 080-099 82 82 82 151 151 151 F F F
96222024-02-27T21:47:41.206Z 5 100-119 87 87 87 166 166 166 F F F
96232024-02-27T21:47:41.206Z 6 120-139 86 86 86 165 165 165 F F F
96242024-02-27T21:47:41.206Z 7 140-159 78 78 78 145 145 145 F F F
96252024-02-27T21:47:41.206Z 8 160-179 88 81 88 167 149 167 F F F <---
96262024-02-27T21:47:41.206Z 9 180-199 86 86 86 163 163 163 F F F
96272024-02-27T21:47:41.206Z 10 200-219 85 85 85 160 160 160 T F T <---
96282024-02-27T21:47:41.206Z 11 220-239 82 82 82 150 150 150 T F T <---
96292024-02-27T21:47:41.207Z 12 240-259 85 85 85 160 160 160 F F F
96302024-02-27T21:47:41.207Z 13 260-279 81 81 81 149 149 149 F F F
96312024-02-27T21:47:41.207Z 14 280-299 88 74 88 167 141 167 F T F <---
96322024-02-27T21:47:41.207Z 15 300-319 82 82 82 150 150 150 F F F
96332024-02-27T21:47:41.207Z 16 320-339 88 84 88 170 155 170 F F F <---
96342024-02-27T21:47:41.207Z 17 340-359 88 87 88 168 166 168 F F F <---
96352024-02-27T21:47:41.207Z 18 360-379 82 82 82 151 151 151 F F F
96362024-02-27T21:47:41.207Z 19 380-399 84 84 84 155 155 155 F F F
96372024-02-27T21:47:41.207Z 20 400-419 88 84 88 167 155 167 F T F <---
96382024-02-27T21:47:41.207Z 21 420-439 86 86 86 165 165 165 F F F
96392024-02-27T21:47:41.208Z 22 440-459 88 86 88 168 161 168 F F F <---
96402024-02-27T21:47:41.208Z 23 460-479 87 87 87 166 166 166 F F F
96412024-02-27T21:47:41.208Z 24 480-499 88 82 88 168 151 168 F F F <---
96422024-02-27T21:47:41.208Z 25 500-519 76 76 76 144 144 144 F F F
96432024-02-27T21:47:41.208Z 26 520-539 83 83 83 153 153 153 F F F
96442024-02-27T21:47:41.208Z 27 540-559 88 86 88 168 165 168 F F F <---
96452024-02-27T21:47:41.208Z 28 560-579 86 86 86 161 161 161 F F F
96462024-02-27T21:47:41.208Z 29 580-599 84 84 84 159 159 159 F F F
96472024-02-27T21:47:41.209ZMax gen: 88, Max flush: 170
96482024-02-27T21:47:41.209ZError: Difference in extent metadata found!
96492024-02-27T21:47:41.209ZA Difference in extent metadata is expected here
96502024-02-27T21:47:41.209ZOn loop 44
96512024-02-27T21:47:41.209Z
96522024-02-27T21:47:41.209ZVerifying data now
96532024-02-27T21:47:45.743ZNo change after verify
96542024-02-27T21:47:45.744ZLoop: 44 Downstairs dump after verify (and repair):
96552024-02-27T21:47:45.748ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96562024-02-27T21:47:45.748Z 0 000-019 89 89 89 171 171 171 F F F
96572024-02-27T21:47:45.748Z 1 020-039 87 87 87 166 166 166 F F F
96582024-02-27T21:47:45.748Z 2 040-059 86 86 86 165 165 165 F F F
96592024-02-27T21:47:45.749Z 3 060-079 87 87 87 166 166 166 F F F
96602024-02-27T21:47:45.749Z 4 080-099 82 82 82 151 151 151 F F F
96612024-02-27T21:47:45.749Z 5 100-119 87 87 87 166 166 166 F F F
96622024-02-27T21:47:45.749Z 6 120-139 86 86 86 165 165 165 F F F
96632024-02-27T21:47:45.749Z 7 140-159 78 78 78 145 145 145 F F F
96642024-02-27T21:47:45.749Z 8 160-179 88 88 88 167 167 167 F F F
96652024-02-27T21:47:45.749Z 9 180-199 86 86 86 163 163 163 F F F
96662024-02-27T21:47:45.749Z 10 200-219 89 89 89 171 171 171 F F F
96672024-02-27T21:47:45.749Z 11 220-239 89 89 89 171 171 171 F F F
96682024-02-27T21:47:45.749Z 12 240-259 85 85 85 160 160 160 F F F
96692024-02-27T21:47:45.750Z 13 260-279 81 81 81 149 149 149 F F F
96702024-02-27T21:47:45.750Z 14 280-299 88 88 88 167 167 167 F F F
96712024-02-27T21:47:45.750Z 15 300-319 82 82 82 150 150 150 F F F
96722024-02-27T21:47:45.750Z 16 320-339 88 88 88 170 170 170 F F F
96732024-02-27T21:47:45.750Z 17 340-359 88 88 88 168 168 168 F F F
96742024-02-27T21:47:45.751Z 18 360-379 82 82 82 151 151 151 F F F
96752024-02-27T21:47:45.751Z 19 380-399 84 84 84 155 155 155 F F F
96762024-02-27T21:47:45.751Z 20 400-419 88 88 88 167 167 167 F F F
96772024-02-27T21:47:45.751Z 21 420-439 86 86 86 165 165 165 F F F
96782024-02-27T21:47:45.751Z 22 440-459 88 88 88 168 168 168 F F F
96792024-02-27T21:47:45.751Z 23 460-479 87 87 87 166 166 166 F F F
96802024-02-27T21:47:45.751Z 24 480-499 88 88 88 168 168 168 F F F
96812024-02-27T21:47:45.751Z 25 500-519 76 76 76 144 144 144 F F F
96822024-02-27T21:47:45.751Z 26 520-539 83 83 83 153 153 153 F F F
96832024-02-27T21:47:45.752Z 27 540-559 88 88 88 168 168 168 F F F
96842024-02-27T21:47:45.752Z 28 560-579 86 86 86 161 161 161 F F F
96852024-02-27T21:47:45.752Z 29 580-599 84 84 84 159 159 159 F F F
96862024-02-27T21:47:45.752ZMax gen: 89, Max flush: 171
96872024-02-27T21:47:45.756Z
96882024-02-27T21:47:45.756ZBegin loop 45
96892024-02-27T21:47:45.757ZDownstairs to restart: 0
96902024-02-27T21:47:45.773Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.159623934Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504}
96912024-02-27T21:47:45.773Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.159884224Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504}
96922024-02-27T21:47:45.773Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.159901112Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504}
96932024-02-27T21:47:45.773Z{"msg":"Crucible bbf9a5d0-c9f0-407c-b0ae-662612f8be3c has session id: a9af836e-f5a5-4292-a5b9-38eac4f1bf13","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.159921499Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
96942024-02-27T21:47:45.773Z{"msg":"Upstairs opts: Upstairs UUID: bbf9a5d0-c9f0-407c-b0ae-662612f8be3c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.159967564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
96952024-02-27T21:47:45.781Z{"msg":"Crucible stats registered with UUID: bbf9a5d0-c9f0-407c-b0ae-662612f8be3c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160028537Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
96962024-02-27T21:47:45.782ZCrucible runtime is spawned
96972024-02-27T21:47:45.782Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160206946Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504}
96982024-02-27T21:47:45.782Z{"msg":"bbf9a5d0-c9f0-407c-b0ae-662612f8be3c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160293935Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
96992024-02-27T21:47:45.782Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160326402Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97002024-02-27T21:47:45.782Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160368946Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97012024-02-27T21:47:45.782Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160411872Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97022024-02-27T21:47:45.783Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160437889Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97032024-02-27T21:47:45.783Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160471814Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97042024-02-27T21:47:45.783Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160495572Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97052024-02-27T21:47:45.783Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160526148Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97062024-02-27T21:47:45.783Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160566883Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97072024-02-27T21:47:45.783Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.16059612Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97082024-02-27T21:47:45.783Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160704357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97092024-02-27T21:47:45.784Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.16076202Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97102024-02-27T21:47:45.784Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.160791577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97112024-02-27T21:47:45.784Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.161483585Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97122024-02-27T21:47:45.784Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.161522601Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97132024-02-27T21:47:45.784Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.161646736Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97142024-02-27T21:47:45.784Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.161679792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97152024-02-27T21:47:45.785Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.161867Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97162024-02-27T21:47:45.785Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.161914044Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97172024-02-27T21:47:45.785Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.162177533Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97182024-02-27T21:47:45.785Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.162220308Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97192024-02-27T21:47:45.785Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.162535361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97202024-02-27T21:47:45.785Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.162781122Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97212024-02-27T21:47:45.785Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.162831307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97222024-02-27T21:47:45.785Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.162855214Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97232024-02-27T21:47:45.785Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.162873962Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97242024-02-27T21:47:45.786Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.162901458Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97252024-02-27T21:47:45.786Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.162924345Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97262024-02-27T21:47:45.786Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163107614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97272024-02-27T21:47:45.786Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.1631432Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97282024-02-27T21:47:45.786Z{"msg":"[0]R flush_numbers[0..12]: [171, 166, 165, 166, 151, 166, 165, 145, 167, 163, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163160138Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97292024-02-27T21:47:45.786Z{"msg":"[0]R generation[0..12]: [89, 87, 86, 87, 82, 87, 86, 78, 88, 86, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163182395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97302024-02-27T21:47:45.787Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163197293Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97312024-02-27T21:47:45.787Z{"msg":"[1]R flush_numbers[0..12]: [171, 166, 165, 166, 151, 166, 165, 145, 167, 163, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163219411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97322024-02-27T21:47:45.787Z{"msg":"[1]R generation[0..12]: [89, 87, 86, 87, 82, 87, 86, 78, 88, 86, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163234029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97332024-02-27T21:47:45.787Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163255486Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":Wait for a query_work_queue command to finish before sending IO
97342024-02-27T21:47:45.787Z"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97352024-02-27T21:47:45.787Z{"msg":"[2]R flush_numbers[0..12]: [171, 166, 165, 166, 151, 166, 165, 145, 167, 163, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163274744Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97362024-02-27T21:47:45.787Z{"msg":"[2]R generation[0..12]: [89, 87, 86, 87, 82, 87, 86, 78, 88, 86, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163299691Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97372024-02-27T21:47:45.787Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163334717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97382024-02-27T21:47:45.787Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163351935Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97392024-02-27T21:47:45.788Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163388361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97402024-02-27T21:47:45.788Z{"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163418307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97412024-02-27T21:47:45.788Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163452003Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"":"downstairs","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97422024-02-27T21:47:45.788Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163490019Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97432024-02-27T21:47:45.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163526745Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"0","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97442024-02-27T21:47:45.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163555561Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"1","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97452024-02-27T21:47:45.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163590587Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"client":"2","":"downstairs","session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97462024-02-27T21:47:45.788Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97472024-02-27T21:47:45.789Z30,"time":"2024-02-27T21:47:44.163634482Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97482024-02-27T21:47:45.789Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163666358Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97492024-02-27T21:47:45.789Z{"msg":"bbf9a5d0-c9f0-407c-b0ae-662612f8be3c is now active with session: a9af836e-f5a5-4292-a5b9-38eac4f1bf13","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163680947Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97502024-02-27T21:47:45.789Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163700344Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504,"session_id":"a9af836e-f5a5-4292-a5b9-38eac4f1bf13"}
97512024-02-27T21:47:45.789Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:44.163713233Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504}
97522024-02-27T21:47:45.789ZLoading write count information from file "/tmp/test_repair_verify.data"
97532024-02-27T21:47:45.789ZRun Repair workload
97542024-02-27T21:47:45.789Z01/30 Write block 7 len 28672 data: 8 8 11 12 12 11 10
97552024-02-27T21:47:45.795Z02/30 Write block 246 len 4096 data: 7
97562024-02-27T21:47:45.795Z03/30 Read block 425 len 12288
97572024-02-27T21:47:45.795Z04/30 Write block 531 len 20480 data: 4 3 4 7 8
97582024-02-27T21:47:45.795Z05/30 Read block 362 len 16384
97592024-02-27T21:47:45.796Z06/30 Read block 21 len 20480
97602024-02-27T21:47:45.796Z07/30 Read block 542 len 28672
97612024-02-27T21:47:45.796Z08/30 Write block 487 len 36864 data: 12 12 14 12 12 10 8 8 5
97622024-02-27T21:47:45.796Z09/30 Read block 389 len 8192
97632024-02-27T21:47:45.796Z10/30 Flush
97642024-02-27T21:47:45.796Z11/30 Write block 120 len 16384 data: 9 9 9 9
97652024-02-27T21:47:45.796Z12/30 Read block 121 len 4096
97662024-02-27T21:47:45.797Z13/30 Flush
97672024-02-27T21:47:45.797Z14/30 Write block 174 len 20480 data: 10 9 10 9 10
97682024-02-27T21:47:45.797Z15/30 Read block 5 len 4096
97692024-02-27T21:47:45.797Z16/30 Write block 396 len 16384 data: 5 8 7 8
97702024-02-27T21:47:45.797Z17/30 Write block 264 len 40960 data: 5 4 3 5 6 6 6 7 7 5
97712024-02-27T21:47:45.801Z18/30 Write block 328 len 12288 data: 5 5 7
97722024-02-27T21:47:45.801Z19/30 Write block 146 len 24576 data: 5 4 4 5 6 6
97732024-02-27T21:47:45.801Z20/30 Write block 157 len 16384 data: 9 9 9 8
97742024-02-27T21:47:45.801Z21/30 Write block 545 len 24576 data: 5 6 6 11 11 10
97752024-02-27T21:47:45.801Z22/30 Flush
97762024-02-27T21:47:45.801Z23/30 Write block 72 len 20480 data: 7 4 5 6 5
97772024-02-27T21:47:45.807Z24/30 Write block 137 len 24576 data: 6 6 7 7 7 8
97782024-02-27T21:47:45.807Z25/30 Read block 408 len 36864
97792024-02-27T21:47:45.807Z26/30 Flush
97802024-02-27T21:47:45.807Z27/30 Write block 509 len 16384 data: 8 10 14 13
97812024-02-27T21:47:45.807Z28/30 Read block 413 len 40960
97822024-02-27T21:47:45.813Z29/30 Write block 123 len 32768 data: 10 10 7 6 6 7 6 6
97832024-02-27T21:47:45.813Z30/30 Write block 179 len 36864 data: 8 10 13 13 11 11 7 8 8
97842024-02-27T21:47:45.813Z----------------------------------------------------------------
97852024-02-27T21:47:45.813Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
97862024-02-27T21:47:45.813ZStates: Active Active Active
97872024-02-27T21:47:45.813ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97882024-02-27T21:47:45.813Z 1 Acked 1000 Write 7 Done Done Done false
97892024-02-27T21:47:45.813Z 2 Acked 1001 Write 1 Done Done Done false
97902024-02-27T21:47:45.814Z 3 Acked 1002 Read 3 Done Done Done false
97912024-02-27T21:47:45.814Z 4 Acked 1003 Write 5 Done Done Done false
97922024-02-27T21:47:45.814Z 5 Acked 1004 Read 4 Done Done Done false
97932024-02-27T21:47:45.814Z 6 Acked 1005 Read 5 Sent Done Done false
97942024-02-27T21:47:45.814Z 7 Acked 1006 Read 7 Sent Done Done false
97952024-02-27T21:47:45.814Z 8 Acked 1007 Write 9 Sent Done Done false
97962024-02-27T21:47:45.814Z 9 Acked 1008 Read 2 Sent Done Done false
97972024-02-27T21:47:45.814Z 10 Acked 1009 Flush 0 Sent Done Done false
97982024-02-27T21:47:45.814Z 11 Acked 1010 Write 4 Sent Done Done false
97992024-02-27T21:47:45.814Z 12 Acked 1011 Read 1 Sent Done Done false
98002024-02-27T21:47:45.814Z 13 Acked 1012 Flush 0 Sent Done Done false
98012024-02-27T21:47:45.814Z 14 Acked 1013 Write 5 Sent Done Done false
98022024-02-27T21:47:45.814Z 15 Acked 1014 Read 1 Sent Done Done false
98032024-02-27T21:47:45.815Z 16 Acked 1015 Write 4 Sent Done Done false
98042024-02-27T21:47:45.815Z 17 Acked 1016 Write 10 Sent Done Done false
98052024-02-27T21:47:45.815Z 18 Acked 1017 Write 3 Sent Done Done false
98062024-02-27T21:47:45.815Z 19 Acked 1018 Write 6 Sent Done Done false
98072024-02-27T21:47:45.815Z 20 Acked 1019 Write 4 Sent Done Done false
98082024-02-27T21:47:45.815Z 21 Acked 1020 Write 6 Sent Done Done false
98092024-02-27T21:47:45.815Z 22 Acked 1021 Flush 0 Sent Done Done false
98102024-02-27T21:47:45.815Z 23 Acked 1022 Write 5 Sent Done Done false
98112024-02-27T21:47:45.815Z 24 Acked 1023 Write 6 Sent Done Done false
98122024-02-27T21:47:45.815Z 25 Acked 1024 Read 9 Sent Done Done false
98132024-02-27T21:47:45.815Z 26 Acked 1025 Flush 0 Sent Done Done false
98142024-02-27T21:47:45.815Z 27 Acked 1026 Write 4 Sent Done Done false
98152024-02-27T21:47:45.815Z 28 Acked 1027 Read 10 Sent Done Done false
98162024-02-27T21:47:45.816Z 29 Acked 1028 Write 8 Sent Done Done false
98172024-02-27T21:47:45.816Z 30 Acked 1029 Write 9 Sent Sent Sent false
98182024-02-27T21:47:45.816Z STATES DS:0 DS:1 DS:2 TOTAL
98192024-02-27T21:47:45.816Z New 0 0 0 0
98202024-02-27T21:47:45.816Z Sent 25 1 1 27
98212024-02-27T21:47:45.816Z Done 5 29 29 63
98222024-02-27T21:47:45.816Z Skipped 0 0 0 0
98232024-02-27T21:47:45.816Z Error 0 0 0 0
98242024-02-27T21:47:45.816ZLast Flush: 0 1025 1025
98252024-02-27T21:47:45.816ZDownstairs last five completed:
98262024-02-27T21:47:45.816ZUpstairs last five completed: 30 29 28 27 26
98272024-02-27T21:47:45.816Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:44.197659914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1504}
98282024-02-27T21:47:45.816ZWrote out file "/tmp/test_repair_verify.data"
98292024-02-27T21:47:45.834Z
98302024-02-27T21:47:45.834ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98312024-02-27T21:47:45.838ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98322024-02-27T21:47:45.838Z 0 000-019 89 90 90 171 172 172 T F F <---
98332024-02-27T21:47:45.838Z 1 020-039 87 87 87 166 166 166 F F F
98342024-02-27T21:47:45.839Z 2 040-059 86 86 86 165 165 165 F F F
98352024-02-27T21:47:45.839Z 3 060-079 87 90 90 166 175 175 F F F <---
98362024-02-27T21:47:45.839Z 4 080-099 82 82 82 151 151 151 F F F
98372024-02-27T21:47:45.839Z 5 100-119 87 87 87 166 166 166 F F F
98382024-02-27T21:47:45.839Z 6 120-139 86 90 90 165 175 175 F T T <---
98392024-02-27T21:47:45.839Z 7 140-159 78 90 90 145 175 175 F F F <---
98402024-02-27T21:47:45.839Z 8 160-179 88 90 90 167 174 174 F T T <---
98412024-02-27T21:47:45.839Z 9 180-199 86 86 86 163 163 163 F T T <---
98422024-02-27T21:47:45.839Z 10 200-219 89 89 89 171 171 171 F F F
98432024-02-27T21:47:45.840Z 11 220-239 89 89 89 171 171 171 F F F
98442024-02-27T21:47:45.840Z 12 240-259 85 90 90 160 172 172 T F F <---
98452024-02-27T21:47:45.840Z 13 260-279 81 90 90 149 174 174 F F F <---
98462024-02-27T21:47:45.840Z 14 280-299 88 88 88 167 167 167 F F F
98472024-02-27T21:47:45.840Z 15 300-319 82 82 82 150 150 150 F F F
98482024-02-27T21:47:45.840Z 16 320-339 88 90 90 170 174 174 F F F <---
98492024-02-27T21:47:45.840Z 17 340-359 88 88 88 168 168 168 F F F
98502024-02-27T21:47:45.840Z 18 360-379 82 82 82 151 151 151 F F F
98512024-02-27T21:47:45.840Z 19 380-399 84 90 90 155 174 174 F F F <---
98522024-02-27T21:47:45.840Z 20 400-419 88 88 88 167 167 167 F F F
98532024-02-27T21:47:45.841Z 21 420-439 86 86 86 165 165 165 F F F
98542024-02-27T21:47:45.841Z 22 440-459 88 88 88 168 168 168 F F F
98552024-02-27T21:47:45.841Z 23 460-479 87 87 87 166 166 166 F F F
98562024-02-27T21:47:45.841Z 24 480-499 88 90 90 168 172 172 F F F <---
98572024-02-27T21:47:45.841Z 25 500-519 76 76 76 144 144 144 F T T <---
98582024-02-27T21:47:45.841Z 26 520-539 83 90 90 153 172 172 T F F <---
98592024-02-27T21:47:45.841Z 27 540-559 88 90 90 168 174 174 F F F <---
98602024-02-27T21:47:45.841Z 28 560-579 86 86 86 161 161 161 F F F
98612024-02-27T21:47:45.842Z 29 580-599 84 84 84 159 159 159 F F F
98622024-02-27T21:47:45.842ZMax gen: 90, Max flush: 175
98632024-02-27T21:47:45.842ZError: Difference in extent metadata found!
98642024-02-27T21:47:45.842ZA Difference in extent metadata is expected here
98652024-02-27T21:47:45.842ZOn loop 45
98662024-02-27T21:47:45.842Z
98672024-02-27T21:47:45.851ZVerifying data now
98682024-02-27T21:47:50.963ZNo change after verify
98692024-02-27T21:47:50.963ZLoop: 45 Downstairs dump after verify (and repair):
98702024-02-27T21:47:50.972ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98712024-02-27T21:47:50.972Z 0 000-019 90 90 90 172 172 172 F F F
98722024-02-27T21:47:50.972Z 1 020-039 87 87 87 166 166 166 F F F
98732024-02-27T21:47:50.973Z 2 040-059 86 86 86 165 165 165 F F F
98742024-02-27T21:47:50.973Z 3 060-079 90 90 90 175 175 175 F F F
98752024-02-27T21:47:50.973Z 4 080-099 82 82 82 151 151 151 F F F
98762024-02-27T21:47:50.973Z 5 100-119 87 87 87 166 166 166 F F F
98772024-02-27T21:47:50.973Z 6 120-139 91 91 91 176 176 176 F F F
98782024-02-27T21:47:50.973Z 7 140-159 90 90 90 175 175 175 F F F
98792024-02-27T21:47:50.973Z 8 160-179 91 91 91 176 176 176 F F F
98802024-02-27T21:47:50.974Z 9 180-199 91 91 91 176 176 176 F F F
98812024-02-27T21:47:50.974Z 10 200-219 89 89 89 171 171 171 F F F
98822024-02-27T21:47:50.974Z 11 220-239 89 89 89 171 171 171 F F F
98832024-02-27T21:47:50.974Z 12 240-259 90 90 90 172 172 172 F F F
98842024-02-27T21:47:50.974Z 13 260-279 90 90 90 174 174 174 F F F
98852024-02-27T21:47:50.974Z 14 280-299 88 88 88 167 167 167 F F F
98862024-02-27T21:47:50.975Z 15 300-319 82 82 82 150 150 150 F F F
98872024-02-27T21:47:50.975Z 16 320-339 90 90 90 174 174 174 F F F
98882024-02-27T21:47:50.975Z 17 340-359 88 88 88 168 168 168 F F F
98892024-02-27T21:47:50.975Z 18 360-379 82 82 82 151 151 151 F F F
98902024-02-27T21:47:50.975Z 19 380-399 90 90 90 174 174 174 F F F
98912024-02-27T21:47:50.975Z 20 400-419 88 88 88 167 167 167 F F F
98922024-02-27T21:47:50.975Z 21 420-439 86 86 86 165 165 165 F F F
98932024-02-27T21:47:50.976Z 22 440-459 88 88 88 168 168 168 F F F
98942024-02-27T21:47:50.976Z 23 460-479 87 87 87 166 166 166 F F F
98952024-02-27T21:47:50.976Z 24 480-499 90 90 90 172 172 172 F F F
98962024-02-27T21:47:50.976Z 25 500-519 91 91 91 176 176 176 F F F
98972024-02-27T21:47:50.976Z 26 520-539 90 90 90 172 172 172 F F F
98982024-02-27T21:47:50.976Z 27 540-559 90 90 90 174 174 174 F F F
98992024-02-27T21:47:50.976Z 28 560-579 86 86 86 161 161 161 F F F
99002024-02-27T21:47:50.976Z 29 580-599 84 84 84 159 159 159 F F F
99012024-02-27T21:47:50.977ZMax gen: 91, Max flush: 176
99022024-02-27T21:47:50.977Z
99032024-02-27T21:47:50.977ZBegin loop 46
99042024-02-27T21:47:50.977ZDownstairs to restart: 2
99052024-02-27T21:47:50.977Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.97050238Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512}
99062024-02-27T21:47:50.977Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
99072024-02-27T21:47:50.977Z2024-02-27T21:47:48.970815933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512}
99082024-02-27T21:47:50.977Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.970864478Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512}
99092024-02-27T21:47:50.978Z{"msg":"Crucible 2ddd69cf-5d67-413b-a11b-06495df2629c has session id: 8830ae39-7ba7-46a6-856f-6b2adc0a8b26","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.970893764Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99102024-02-27T21:47:50.978Z{"msg":"Upstairs opts: Upstairs UUID: 2ddd69cf-5d67-413b-a11b-06495df2629c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.970917461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99112024-02-27T21:47:50.978Z{"msg":"Crucible stats registered with UUID: 2ddd69cf-5d67-413b-a11b-06495df2629c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.970942338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99122024-02-27T21:47:50.978Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.970955607Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512}
99132024-02-27T21:47:50.978Z{"msg":"2ddd69cf-5d67-413b-a11b-06495df2629c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.970972155Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99142024-02-27T21:47:50.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.970992312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99152024-02-27T21:47:50.978Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971018459Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99162024-02-27T21:47:50.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971041437Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99172024-02-27T21:47:50.979Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971063964Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99182024-02-27T21:47:50.979Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971091021Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99192024-02-27T21:47:50.979Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971108639Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99202024-02-27T21:47:50.980Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971191339Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99212024-02-27T21:47:50.980Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971295147Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99222024-02-27T21:47:50.980Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971366608Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99232024-02-27T21:47:50.980Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971411103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99242024-02-27T21:47:50.980Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.971458177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99252024-02-27T21:47:50.980Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.97160354Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99262024-02-27T21:47:50.980Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.972277611Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99272024-02-27T21:47:50.980Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.972326535Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99282024-02-27T21:47:50.980Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.972357811Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99292024-02-27T21:47:50.980Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.972405616Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99302024-02-27T21:47:50.981Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.972497725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99312024-02-27T21:47:50.981Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.972535291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99322024-02-27T21:47:50.981Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973014224Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99332024-02-27T21:47:50.981Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973057449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99342024-02-27T21:47:50.981Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973075317Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99352024-02-27T21:47:50.981Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973592026Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99362024-02-27T21:47:50.981Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973635601Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99372024-02-27T21:47:50.981Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973661008Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99382024-02-27T21:47:50.981Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973803231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99392024-02-27T21:47:50.982Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973840517Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99402024-02-27T21:47:50.982Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973856355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99412024-02-27T21:47:50.982Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973887651Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99422024-02-27T21:47:50.982Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973919308Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99432024-02-27T21:47:50.982Z{"msg":"[0]R flush_numbers[0..12]: [172, 166, 165, 175, 151, 166, 176, 175, 176, 176, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973937505Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99442024-02-27T21:47:50.982Z{"msg":"[0]R generation[0..12]: [90, 87, 86, 90, 82, 87, 91, 90, 91, 91, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.973969182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99452024-02-27T21:47:50.982Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
99462024-02-27T21:47:50.982Z2024-02-27T21:47:48.974005908Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99472024-02-27T21:47:50.982Z{"msg":"[1]R flush_numbers[0..12]: [172, 166, 165, 175, 151, 166, 176, 175, 176, 176, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974044123Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99482024-02-27T21:47:50.983Z{"msg":"[1]R generation[0..12]: [90, 87, 86, 90, 82, 87, 91, 90, 91, 91, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.97406806Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99492024-02-27T21:47:50.983Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974097067Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99502024-02-27T21:47:50.983Z{"msg":"[2]R flush_numbers[0..12]: [172, 166, 165, 175, 151, 166, 176, 175, 176, 176, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974113355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99512024-02-27T21:47:50.983Z{"msg":"[2]R generation[0..12]: [90, 87, 86, 90, 82, 87, 91, 90, 91, 91, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974135782Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99522024-02-27T21:47:50.983Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974161899Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99532024-02-27T21:47:50.983Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974176097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99542024-02-27T21:47:50.983Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974196775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99552024-02-27T21:47:50.983Z{"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974223992Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99562024-02-27T21:47:50.983Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.9742382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"":"downstairs","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99572024-02-27T21:47:50.983Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974258868Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99582024-02-27T21:47:50.983Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974285015Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"0","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99592024-02-27T21:47:50.984Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974299513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"1","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99602024-02-27T21:47:50.984Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.97432065Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"client":"2","":"downstairs","session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99612024-02-27T21:47:50.984Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974346997Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99622024-02-27T21:47:50.984Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974360756Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99632024-02-27T21:47:50.984Z{"msg":"2ddd69cf-5d67-413b-a11b-06495df2629c is now active with session: 8830ae39-7ba7-46a6-856f-6b2adc0a8b26","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974389772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99642024-02-27T21:47:50.984Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99652024-02-27T21:47:50.984ZSet Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974427158Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512,"session_id":"8830ae39-7ba7-46a6-856f-6b2adc0a8b26"}
99662024-02-27T21:47:50.984Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:48.974447316Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512}
99672024-02-27T21:47:50.984ZLoading write count information from file "/tmp/test_repair_verify.data"
99682024-02-27T21:47:50.985ZRun Repair workload
99692024-02-27T21:47:50.985Z01/30 Write block 56 len 12288 data: 11 11 9
99702024-02-27T21:47:50.991Z02/30 Read block 238 len 4096
99712024-02-27T21:47:50.991Z03/30 Flush
99722024-02-27T21:47:50.991Z04/30 Write block 428 len 24576 data: 6 8 8 8 8 10
99732024-02-27T21:47:50.991Z05/30 Read block 443 len 24576
99742024-02-27T21:47:50.991Z06/30 Write block 583 len 32768 data: 5 5 4 5 6 3 6 6
99752024-02-27T21:47:50.991Z07/30 Read block 78 len 8192
99762024-02-27T21:47:50.992Z08/30 Read block 370 len 20480
99772024-02-27T21:47:50.992Z09/30 Write block 271 len 24576 data: 8 8 6 6 4 6
99782024-02-27T21:47:50.992Z10/30 Write block 265 len 24576 data: 5 4 6 7 7 7
99792024-02-27T21:47:50.992Z11/30 Read block 310 len 24576
99802024-02-27T21:47:50.992Z12/30 Read block 178 len 20480
99812024-02-27T21:47:50.992Z13/30 Write block 447 len 32768 data: 5 6 6 7 8 7 8 6
99822024-02-27T21:47:50.992Z14/30 Flush
99832024-02-27T21:47:50.992Z15/30 Write block 123 len 36864 data: 11 11 8 7 7 8 7 7 5
99842024-02-27T21:47:50.993Z16/30 Write block 325 len 36864 data: 5 4 5 6 6 8 8 8 9
99852024-02-27T21:47:50.993Z17/30 Read block 8 len 36864
99862024-02-27T21:47:50.993Z18/30 Write block 353 len 28672 data: 8 8 11 10 6 6 8
99872024-02-27T21:47:50.993Z19/30 Write block 126 len 32768 data: 8 8 9 8 8 6 3 4
99882024-02-27T21:47:50.993Z20/30 Write block 390 len 36864 data: 8 9 7 8 6 6 6 9 8
99892024-02-27T21:47:50.993Z21/30 Read block 20 len 20480
99902024-02-27T21:47:50.993Z22/30 Read block 527 len 40960
99912024-02-27T21:47:50.994Z23/30 Write block 583 len 16384 data: 6 6 5 6
99922024-02-27T21:47:50.994Z24/30 Read block 443 len 12288
99932024-02-27T21:47:50.994Z25/30 Read block 230 len 24576
99942024-02-27T21:47:50.994Z26/30 Read block 353 len 8192
99952024-02-27T21:47:50.994Z27/30 Read block 550 len 20480
99962024-02-27T21:47:50.994Z28/30 Write block 103 len 24576 data: 7 6 8 9 12 11
99972024-02-27T21:47:50.994Z29/30 Write block 180 len 4096 data: 11
99982024-02-27T21:47:50.995Z30/30 Read block 584 len 8192
99992024-02-27T21:47:50.995Z----------------------------------------------------------------
100002024-02-27T21:47:50.995Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
100012024-02-27T21:47:50.995ZStates: Active Active Active
100022024-02-27T21:47:50.995ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
100032024-02-27T21:47:50.995Z 1 Acked 1000 Write 3 Done Done Done false
100042024-02-27T21:47:50.995Z 2 Acked 1001 Read 1 Done Done Sent false
100052024-02-27T21:47:50.995Z 3 Acked 1002 Flush 0 Done Done Sent false
100062024-02-27T21:47:50.996Z 4 Acked 1003 Write 6 Done Done Sent false
100072024-02-27T21:47:50.996Z 5 Acked 1004 Read 6 Done Done Sent false
100082024-02-27T21:47:50.996Z 6 Acked 1005 Write 8 Done Done Sent false
100092024-02-27T21:47:50.996Z 7 Acked 1006 Read 2 Done Done Sent false
100102024-02-27T21:47:50.996Z 8 Acked 1007 Read 5 Done Done Sent false
100112024-02-27T21:47:50.996Z 9 Acked 1008 Write 6 Done Done Sent false
100122024-02-27T21:47:50.996Z 10 Acked 1009 Write 6 Done Done Sent false
100132024-02-27T21:47:50.997Z 11 Acked 1010 Read 6 Done Done Sent false
100142024-02-27T21:47:50.997Z 12 Acked 1011 Read 5 Done Done Sent false
100152024-02-27T21:47:50.997Z 13 Acked 1012 Write 8 Done Done Sent false
100162024-02-27T21:47:50.997Z 14 Acked 1013 Flush 0 Done Done Sent false
100172024-02-27T21:47:50.997Z 15 Acked 1014 Write 9 Done Done Sent false
100182024-02-27T21:47:50.997Z 16 Acked 1015 Write 9 Done Done Sent false
100192024-02-27T21:47:50.997Z 17 Acked 1016 Read 9 Done Done Sent false
100202024-02-27T21:47:50.997Z 18 Acked 1017 Write 7 Done Done Sent false
100212024-02-27T21:47:50.997Z 19 Acked 1018 Write 8 Done Done Sent false
100222024-02-27T21:47:50.998Z 20 Acked 1019 Write 9 Done Done Sent false
100232024-02-27T21:47:50.998Z 21 Acked 1020 Read 5 Done Done Sent false
100242024-02-27T21:47:50.998Z 22 Acked 1021 Read 10 Done Done Sent false
100252024-02-27T21:47:50.998Z 23 Acked 1022 Write 4 Done Done Sent false
100262024-02-27T21:47:50.998Z 24 Acked 1023 Read 3 Done Done Sent false
100272024-02-27T21:47:50.998Z 25 Acked 1024 Read 6 Done Done Sent false
100282024-02-27T21:47:50.998Z 26 Acked 1025 Read 2 Done Done Sent false
100292024-02-27T21:47:50.999Z 27 Acked 1026 Read 5 Done Done Sent false
100302024-02-27T21:47:50.999Z 28 Acked 1027 Write 6 Done Done Sent false
100312024-02-27T21:47:50.999Z 29 Acked 1028 Write 1 Done Done Sent false
100322024-02-27T21:47:50.999Z 30 Acked 1029 Read 2 Done Done Sent false
100332024-02-27T21:47:50.999Z STATES DS:0 DS:1 DS:2 TOTAL
100342024-02-27T21:47:50.999Z New 0 0 0 0
100352024-02-27T21:47:50.999Z Sent 0 0 29 29
100362024-02-27T21:47:51.000Z Done 30 30 1 61
100372024-02-27T21:47:51.000Z Skipped 0 0 0 0
100382024-02-27T21:47:51.000Z Error 0 0 0 0
100392024-02-27T21:47:51.000ZLast Flush: 1013 1013 0
100402024-02-27T21:47:51.000ZDownstairs last five completed:
100412024-02-27T21:47:51.000ZUpstairs last five completed: 30 29 28 27 26
100422024-02-27T21:47:51.000Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:49.006152981Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1512}
100432024-02-27T21:47:51.000ZWrote out file "/tmp/test_repair_verify.data"
100442024-02-27T21:47:51.000Z
100452024-02-27T21:47:51.000ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100462024-02-27T21:47:51.002ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100472024-02-27T21:47:51.002Z 0 000-019 90 90 90 172 172 172 F F F
100482024-02-27T21:47:51.002Z 1 020-039 87 87 87 166 166 166 F F F
100492024-02-27T21:47:51.002Z 2 040-059 92 92 86 177 177 165 F F T <---
100502024-02-27T21:47:51.002Z 3 060-079 90 90 90 175 175 175 F F F
100512024-02-27T21:47:51.002Z 4 080-099 82 82 82 151 151 151 F F F
100522024-02-27T21:47:51.002Z 5 100-119 87 87 87 166 166 166 T T F <---
100532024-02-27T21:47:51.003Z 6 120-139 91 91 91 176 176 176 T T F <---
100542024-02-27T21:47:51.003Z 7 140-159 90 90 90 175 175 175 F F F
100552024-02-27T21:47:51.003Z 8 160-179 91 91 91 176 176 176 F F F
100562024-02-27T21:47:51.003Z 9 180-199 91 91 91 176 176 176 T T F <---
100572024-02-27T21:47:51.003Z 10 200-219 89 89 89 171 171 171 F F F
100582024-02-27T21:47:51.003Z 11 220-239 89 89 89 171 171 171 F F F
100592024-02-27T21:47:51.003Z 12 240-259 90 90 90 172 172 172 F F F
100602024-02-27T21:47:51.003Z 13 260-279 92 92 90 178 178 174 F F F <---
100612024-02-27T21:47:51.003Z 14 280-299 88 88 88 167 167 167 F F F
100622024-02-27T21:47:51.003Z 15 300-319 82 82 82 150 150 150 F F F
100632024-02-27T21:47:51.003Z 16 320-339 90 90 90 174 174 174 T T F <---
100642024-02-27T21:47:51.003Z 17 340-359 88 88 88 168 168 168 T T F <---
100652024-02-27T21:47:51.003Z 18 360-379 82 82 82 151 151 151 F F F
100662024-02-27T21:47:51.004Z 19 380-399 90 90 90 174 174 174 T T F <---
100672024-02-27T21:47:51.004Z 20 400-419 88 88 88 167 167 167 F F F
100682024-02-27T21:47:51.004Z 21 420-439 92 92 86 178 178 165 F F F <---
100692024-02-27T21:47:51.004Z 22 440-459 92 92 88 178 178 168 F F F <---
100702024-02-27T21:47:51.008Z 23 460-479 87 87 87 166 166 166 F F F
100712024-02-27T21:47:51.009Z 24 480-499 90 90 90 172 172 172 F F F
100722024-02-27T21:47:51.009Z 25 500-519 91 91 91 176 176 176 F F F
100732024-02-27T21:47:51.009Z 26 520-539 90 90 90 172 172 172 F F F
100742024-02-27T21:47:51.009Z 27 540-559 90 90 90 174 174 174 F F F
100752024-02-27T21:47:51.009Z 28 560-579 86 86 86 161 161 161 F F F
100762024-02-27T21:47:51.009Z 29 580-599 92 92 84 178 178 159 T T F <---
100772024-02-27T21:47:51.009ZMax gen: 92, Max flush: 178
100782024-02-27T21:47:51.010ZError: Difference in extent metadata found!
100792024-02-27T21:47:51.010ZA Difference in extent metadata is expected here
100802024-02-27T21:47:51.010ZOn loop 46
100812024-02-27T21:47:51.010Z
100822024-02-27T21:47:51.010ZVerifying data now
100832024-02-27T21:47:55.233ZNo change after verify
100842024-02-27T21:47:55.233ZLoop: 46 Downstairs dump after verify (and repair):
100852024-02-27T21:47:55.246ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100862024-02-27T21:47:55.246Z 0 000-019 90 90 90 172 172 172 F F F
100872024-02-27T21:47:55.246Z 1 020-039 87 87 87 166 166 166 F F F
100882024-02-27T21:47:55.246Z 2 040-059 92 92 92 177 177 177 F F F
100892024-02-27T21:47:55.247Z 3 060-079 90 90 90 175 175 175 F F F
100902024-02-27T21:47:55.247Z 4 080-099 82 82 82 151 151 151 F F F
100912024-02-27T21:47:55.247Z 5 100-119 93 93 93 179 179 179 F F F
100922024-02-27T21:47:55.247Z 6 120-139 93 93 93 179 179 179 F F F
100932024-02-27T21:47:55.248Z 7 140-159 90 90 90 175 175 175 F F F
100942024-02-27T21:47:55.248Z 8 160-179 91 91 91 176 176 176 F F F
100952024-02-27T21:47:55.248Z 9 180-199 93 93 93 179 179 179 F F F
100962024-02-27T21:47:55.248Z 10 200-219 89 89 89 171 171 171 F F F
100972024-02-27T21:47:55.248Z 11 220-239 89 89 89 171 171 171 F F F
100982024-02-27T21:47:55.253Z 12 240-259 90 90 90 172 172 172 F F F
100992024-02-27T21:47:55.253Z 13 260-279 92 92 92 178 178 178 F F F
101002024-02-27T21:47:55.254Z 14 280-299 88 88 88 167 167 167 F F F
101012024-02-27T21:47:55.254Z 15 300-319 82 82 82 150 150 150 F F F
101022024-02-27T21:47:55.254Z 16 320-339 93 93 93 179 179 179 F F F
101032024-02-27T21:47:55.254Z 17 340-359 93 93 93 179 179 179 F F F
101042024-02-27T21:47:55.254Z 18 360-379 82 82 82 151 151 151 F F F
101052024-02-27T21:47:55.254Z 19 380-399 93 93 93 179 179 179 F F F
101062024-02-27T21:47:55.254Z 20 400-419 88 88 88 167 167 167 F F F
101072024-02-27T21:47:55.254Z 21 420-439 92 92 92 178 178 178 F F F
101082024-02-27T21:47:55.254Z 22 440-459 92 92 92 178 178 178 F F F
101092024-02-27T21:47:55.254Z 23 460-479 87 87 87 166 166 166 F F F
101102024-02-27T21:47:55.255Z 24 480-499 90 90 90 172 172 172 F F F
101112024-02-27T21:47:55.255Z 25 500-519 91 91 91 176 176 176 F F F
101122024-02-27T21:47:55.255Z 26 520-539 90 90 90 172 172 172 F F F
101132024-02-27T21:47:55.255Z 27 540-559 90 90 90 174 174 174 F F F
101142024-02-27T21:47:55.255Z 28 560-579 86 86 86 161 161 161 F F F
101152024-02-27T21:47:55.255Z 29 580-599 93 93 93 179 179 179 F F F
101162024-02-27T21:47:55.255ZMax gen: 93, Max flush: 179
101172024-02-27T21:47:55.255Z
101182024-02-27T21:47:55.255ZBegin loop 47
101192024-02-27T21:47:55.255ZDownstairs to restart: 2
101202024-02-27T21:47:55.267Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.652733679Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520}
101212024-02-27T21:47:55.267Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653019955Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520}
101222024-02-27T21:47:55.280Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653050741Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520}
101232024-02-27T21:47:55.280ZCrucible runtime is spawned
101242024-02-27T21:47:55.280Z{"msg":"Crucible c015dc66-d8ae-468d-815e-6f28c6359b78 has session id: 6a41d852-3160-4554-9acb-a33108fccfa8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653119653Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101252024-02-27T21:47:55.280Z{"msg":"Upstairs opts: Upstairs UUID: c015dc66-d8ae-468d-815e-6f28c6359b78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653189835Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101262024-02-27T21:47:55.280Z{"msg":"Crucible stats registered with UUID: c015dc66-d8ae-468d-815e-6f28c6359b78","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653238139Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101272024-02-27T21:47:55.281Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653278874Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520}
101282024-02-27T21:47:55.281Z{"msg":"c015dc66-d8ae-468d-815e-6f28c6359b78 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.65340094Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101292024-02-27T21:47:55.281Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653466982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101302024-02-27T21:47:55.281Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653524636Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101312024-02-27T21:47:55.281Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.65357119Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101322024-02-27T21:47:55.281Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653602036Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101332024-02-27T21:47:55.281Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653626693Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101342024-02-27T21:47:55.281Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653649851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101352024-02-27T21:47:55.281Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653685166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101362024-02-27T21:47:55.281Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653838559Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101372024-02-27T21:47:55.281Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653890552Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101382024-02-27T21:47:55.281Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653916139Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101392024-02-27T21:47:55.282Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653949015Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101402024-02-27T21:47:55.282Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.653982251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101412024-02-27T21:47:55.282Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.654513159Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101422024-02-27T21:47:55.282Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.654575772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101432024-02-27T21:47:55.282Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.654614107Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101442024-02-27T21:47:55.282Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.654635885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101452024-02-27T21:47:55.282Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.654979024Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101462024-02-27T21:47:55.282Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.65501407Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101472024-02-27T21:47:55.282Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.655156943Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101482024-02-27T21:47:55.282Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.655196679Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101492024-02-27T21:47:55.283Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.655698729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101502024-02-27T21:47:55.283Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.655730716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101512024-02-27T21:47:55.283Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.655756822Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101522024-02-27T21:47:55.283Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.655797698Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101532024-02-27T21:47:55.283Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.655821845Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101542024-02-27T21:47:55.283Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.655850711Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101552024-02-27T21:47:55.283Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.65586588Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101562024-02-27T21:47:55.283Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656210329Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101572024-02-27T21:47:55.283Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656256704Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101582024-02-27T21:47:55.283Z{"msg":"[0]R flush_numbers[0..12]: [172, 166, 177, 175, 151, 179, 179, 175, 176, 179, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656283351Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101592024-02-27T21:47:55.284Z{"msg":"[0]R generation[0..12]: [90, 87, 92, 90, 82, 93, 93, 90, 91, 93, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656307857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101602024-02-27T21:47:55.284Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656343244Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101612024-02-27T21:47:55.284Z{"msg":"[1]R flush_numbers[0..12]: [172, 166, 177, 175, 151, 179, 179, 175, 176, 179, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656380349Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101622024-02-27T21:47:55.284Z{"msg":"[1]R generation[0..12]: [90, 87, 92, 90, 82, 93, 93, 90, 91, 93, 89, 89]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
101632024-02-27T21:47:55.284Z,"time":"2024-02-27T21:47:53.656406826Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101642024-02-27T21:47:55.284Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656444181Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101652024-02-27T21:47:55.284Z{"msg":"[2]R flush_numbers[0..12]: [172, 166, 177, 175, 151, 179, 179, 175, 176, 179, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656483897Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101662024-02-27T21:47:55.284Z{"msg":"[2]R generation[0..12]: [90, 87, 92, 90, 82, 93, 93, 90, 91, 93, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656520412Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101672024-02-27T21:47:55.284Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656550339Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101682024-02-27T21:47:55.284Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656584865Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101692024-02-27T21:47:55.284Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656610862Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101702024-02-27T21:47:55.284Z{"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656634319Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101712024-02-27T21:47:55.285Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656648488Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"":"downstairs","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101722024-02-27T21:47:55.285Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656669515Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101732024-02-27T21:47:55.285Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656682364Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"0","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101742024-02-27T21:47:55.285Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656695812Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"1","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101752024-02-27T21:47:55.285Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.65671594Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"client":"2","":"downstairs","session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101762024-02-27T21:47:55.285Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656729888Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101772024-02-27T21:47:55.285Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656749696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101782024-02-27T21:47:55.285Z{"msg":"c015dc66-d8ae-468d-815e-6f28c6359b78 is now active with session: 6a41d852-3160-4554-9acb-a33108fccfa8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656762804Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101792024-02-27T21:47:55.286Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656784701Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101802024-02-27T21:47:55.286Zip-10-150-1-196.us-west-2.compute.internal","pid":1520,"session_id":"6a41d852-3160-4554-9acb-a33108fccfa8"}
101812024-02-27T21:47:55.286Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:53.656813078Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520}
101822024-02-27T21:47:55.287ZLoading write count information from file "/tmp/test_repair_verify.data"
101832024-02-27T21:47:55.287ZRun Repair workload
101842024-02-27T21:47:55.287Z01/30 Write block 379 len 4096 data: 4
101852024-02-27T21:47:55.287Z02/30 Read block 215 len 20480
101862024-02-27T21:47:55.287Z03/30 Flush
101872024-02-27T21:47:55.287Z04/30 Write block 98 len 24576 data: 5 7 8 8 8 8
101882024-02-27T21:47:55.294Z05/30 Read block 501 len 20480
101892024-02-27T21:47:55.295Z06/30 Read block 249 len 28672
101902024-02-27T21:47:55.295Z07/30 Read block 553 len 40960
101912024-02-27T21:47:55.295Z08/30 Write block 504 len 12288 data: 5 6 7
101922024-02-27T21:47:55.295Z09/30 Read block 81 len 40960
101932024-02-27T21:47:55.295Z10/30 Write block 178 len 20480 data: 11 9 12 14 14
101942024-02-27T21:47:55.295Z11/30 Read block 233 len 20480
101952024-02-27T21:47:55.295Z12/30 Read block 105 len 32768
101962024-02-27T21:47:55.296Z13/30 Flush
101972024-02-27T21:47:55.296Z14/30 Write block 139 len 40960 data: 8 8 8 9 8 6 5 6 5 5
101982024-02-27T21:47:55.296Z15/30 Write block 113 len 4096 data: 7
101992024-02-27T21:47:55.296Z16/30 Write block 314 len 32768 data: 8 6 9 10 10 12 11 9
102002024-02-27T21:47:55.296Z17/30 Write block 582 len 36864 data: 7 7 7 6 7 7 4 7 7
102012024-02-27T21:47:55.296Z18/30 Read block 76 len 8192
102022024-02-27T21:47:55.297Z19/30 Flush
102032024-02-27T21:47:55.301Z20/30 Write block 275 len 20480 data: 5 7 6 6 4
102042024-02-27T21:47:55.301Z21/30 Read block 499 len 40960
102052024-02-27T21:47:55.301Z22/30 Read block 425 len 32768
102062024-02-27T21:47:55.301Z23/30 Read block 318 len 36864
102072024-02-27T21:47:55.301Z24/30 Flush
102082024-02-27T21:47:55.302Z25/30 Write block 144 len 28672 data: 7 6 7 6 6 6 7
102092024-02-27T21:47:55.302Z26/30 Read block 415 len 40960
102102024-02-27T21:47:55.306Z27/30 Read block 131 len 24576
102112024-02-27T21:47:55.306Z28/30 Read block 240 len 20480
102122024-02-27T21:47:55.306Z29/30 Write block 482 len 24576 data: 10 12 12 12 13 13
102132024-02-27T21:47:55.307Z30/30 Read block 139 len 16384
102142024-02-27T21:47:55.307Z----------------------------------------------------------------
102152024-02-27T21:47:55.307Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
102162024-02-27T21:47:55.307ZStates: Active Active Active
102172024-02-27T21:47:55.307ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102182024-02-27T21:47:55.307Z 1 Acked 1000 Write 1 Done Done Done false
102192024-02-27T21:47:55.307Z 2 Acked 1001 Read 5 Done Done Done false
102202024-02-27T21:47:55.308Z 3 Acked 1002 Flush 0 Done Done Sent false
102212024-02-27T21:47:55.308Z 4 Acked 1003 Write 6 Done Done Sent false
102222024-02-27T21:47:55.308Z 5 Acked 1004 Read 5 Done Done Sent false
102232024-02-27T21:47:55.308Z 6 Acked 1005 Read 7 Done Done Sent false
102242024-02-27T21:47:55.308Z 7 Acked 1006 Read 10 Done Done Sent false
102252024-02-27T21:47:55.308Z 8 Acked 1007 Write 3 Done Done Sent false
102262024-02-27T21:47:55.308Z 9 Acked 1008 Read 10 Done Done Sent false
102272024-02-27T21:47:55.309Z 10 Acked 1009 Write 5 Done Done Sent false
102282024-02-27T21:47:55.309Z 11 Acked 1010 Read 5 Done Done Sent false
102292024-02-27T21:47:55.309Z 12 Acked 1011 Read 8 Done Done Sent false
102302024-02-27T21:47:55.309Z 13 Acked 1012 Flush 0 Done Done Sent false
102312024-02-27T21:47:55.309Z 14 Acked 1013 Write 10 Done Done Sent false
102322024-02-27T21:47:55.309Z 15 Acked 1014 Write 1 Done Done Sent false
102332024-02-27T21:47:55.310Z 16 Acked 1015 Write 8 Done Done Sent false
102342024-02-27T21:47:55.310Z 17 Acked 1016 Write 9 Done Done Sent false
102352024-02-27T21:47:55.310Z 18 Acked 1017 Read 2 Done Done Sent false
102362024-02-27T21:47:55.310Z 19 Acked 1018 Flush 0 Done Done Sent false
102372024-02-27T21:47:55.310Z 20 Acked 1019 Write 5 Done Done Sent false
102382024-02-27T21:47:55.311Z 21 Acked 1020 Read 10 Done Done Sent false
102392024-02-27T21:47:55.311Z 22 Acked 1021 Read 8 Done Done Sent false
102402024-02-27T21:47:55.311Z 23 Acked 1022 Read 9 Done Done Sent false
102412024-02-27T21:47:55.311Z 24 Acked 1023 Flush 0 Done Done Sent false
102422024-02-27T21:47:55.311Z 25 Acked 1024 Write 7 Done Done Sent false
102432024-02-27T21:47:55.311Z 26 Acked 1025 Read 10 Done Done Sent false
102442024-02-27T21:47:55.311Z 27 Acked 1026 Read 6 Done Done Sent false
102452024-02-27T21:47:55.312Z 28 Acked 1027 Read 5 Done Done Sent false
102462024-02-27T21:47:55.312Z 29 Acked 1028 Write 6 Done Done Sent false
102472024-02-27T21:47:55.312Z 30 Acked 1029 Read 4 Done Done Sent false
102482024-02-27T21:47:55.312Z STATES DS:0 DS:1 DS:2 TOTAL
102492024-02-27T21:47:55.312Z New 0 0 0 0
102502024-02-27T21:47:55.312Z Sent 0 0 28 28
102512024-02-27T21:47:55.313Z Done 30 30 2 62
102522024-02-27T21:47:55.313Z Skipped 0 0 0 0
102532024-02-27T21:47:55.313Z Error 0 0 0 0
102542024-02-27T21:47:55.313ZLast Flush: 1023 1023 0
102552024-02-27T21:47:55.313ZDownstairs last five completed:
102562024-02-27T21:47:55.313ZUpstairs last five completed: 30 29 28 27 26
102572024-02-27T21:47:55.313Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:53.692055527Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1520}
102582024-02-27T21:47:55.318ZWrote out file "/tmp/test_repair_verify.data"
102592024-02-27T21:47:55.319Z
102602024-02-27T21:47:55.319ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102612024-02-27T21:47:55.329ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102622024-02-27T21:47:55.330Z 0 000-019 90 90 90 172 172 172 F F F
102632024-02-27T21:47:55.330Z 1 020-039 87 87 87 166 166 166 F F F
102642024-02-27T21:47:55.330Z 2 040-059 92 92 92 177 177 177 F F F
102652024-02-27T21:47:55.330Z 3 060-079 90 90 90 175 175 175 F F F
102662024-02-27T21:47:55.330Z 4 080-099 94 94 82 181 181 151 F F F <---
102672024-02-27T21:47:55.330Z 5 100-119 94 94 93 182 182 179 F F F <---
102682024-02-27T21:47:55.331Z 6 120-139 94 94 93 182 182 179 F F F <---
102692024-02-27T21:47:55.331Z 7 140-159 94 94 90 182 182 175 T T F <---
102702024-02-27T21:47:55.335Z 8 160-179 94 94 91 181 181 176 F F F <---
102712024-02-27T21:47:55.335Z 9 180-199 94 94 93 181 181 179 F F F <---
102722024-02-27T21:47:55.336Z 10 200-219 89 89 89 171 171 171 F F F
102732024-02-27T21:47:55.336Z 11 220-239 89 89 89 171 171 171 F F F
102742024-02-27T21:47:55.336Z 12 240-259 90 90 90 172 172 172 F F F
102752024-02-27T21:47:55.336Z 13 260-279 94 94 92 183 183 178 F F F <---
102762024-02-27T21:47:55.336Z 14 280-299 88 88 88 167 167 167 F F F
102772024-02-27T21:47:55.336Z 15 300-319 94 94 82 182 182 150 F F F <---
102782024-02-27T21:47:55.337Z 16 320-339 94 94 93 182 182 179 F F F <---
102792024-02-27T21:47:55.337Z 17 340-359 93 93 93 179 179 179 F F F
102802024-02-27T21:47:55.337Z 18 360-379 94 94 82 180 180 151 F F T <---
102812024-02-27T21:47:55.337Z 19 380-399 93 93 93 179 179 179 F F F
102822024-02-27T21:47:55.337Z 20 400-419 88 88 88 167 167 167 F F F
102832024-02-27T21:47:55.337Z 21 420-439 92 92 92 178 178 178 F F F
102842024-02-27T21:47:55.337Z 22 440-459 92 92 92 178 178 178 F F F
102852024-02-27T21:47:55.337Z 23 460-479 87 87 87 166 166 166 F F F
102862024-02-27T21:47:55.337Z 24 480-499 90 90 90 172 172 172 T T F <---
102872024-02-27T21:47:55.338Z 25 500-519 94 94 91 181 181 176 F F F <---
102882024-02-27T21:47:55.338Z 26 520-539 90 90 90 172 172 172 F F F
102892024-02-27T21:47:55.338Z 27 540-559 90 90 90 174 174 174 F F F
102902024-02-27T21:47:55.338Z 28 560-579 86 86 86 161 161 161 F F F
102912024-02-27T21:47:55.338Z 29 580-599 94 94 93 182 182 179 F F F <---
102922024-02-27T21:47:55.338ZMax gen: 94, Max flush: 183
102932024-02-27T21:47:55.338ZError: Difference in extent metadata found!
102942024-02-27T21:47:55.338ZA Difference in extent metadata is expected here
102952024-02-27T21:47:55.338ZOn loop 47
102962024-02-27T21:47:55.338Z
102972024-02-27T21:47:55.338ZVerifying data now
102982024-02-27T21:47:59.981ZNo change after verify
102992024-02-27T21:47:59.981ZLoop: 47 Downstairs dump after verify (and repair):
103002024-02-27T21:48:00.007ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103012024-02-27T21:48:00.007Z 0 000-019 90 90 90 172 172 172 F F F
103022024-02-27T21:48:00.007Z 1 020-039 87 87 87 166 166 166 F F F
103032024-02-27T21:48:00.008Z 2 040-059 92 92 92 177 177 177 F F F
103042024-02-27T21:48:00.008Z 3 060-079 90 90 90 175 175 175 F F F
103052024-02-27T21:48:00.008Z 4 080-099 94 94 94 181 181 181 F F F
103062024-02-27T21:48:00.008Z 5 100-119 94 94 94 182 182 182 F F F
103072024-02-27T21:48:00.008Z 6 120-139 94 94 94 182 182 182 F F F
103082024-02-27T21:48:00.008Z 7 140-159 95 95 95 184 184 184 F F F
103092024-02-27T21:48:00.427Z 8 160-179 94 94 94 181 181 181 F F F
103102024-02-27T21:48:00.428Z 9 180-199 94 94 94 181 181 181 F F F
103112024-02-27T21:48:00.428Z 10 200-219 89 89 89 171 171 171 F F F
103122024-02-27T21:48:00.428Z 11 220-239 89 89 89 171 171 171 F F F
103132024-02-27T21:48:00.428Z 12 240-259 90 90 90 172 172 172 F F F
103142024-02-27T21:48:00.428Z 13 260-279 94 94 94 183 183 183 F F F
103152024-02-27T21:48:00.428Z 14 280-299 88 88 88 167 167 167 F F F
103162024-02-27T21:48:00.429Z 15 300-319 94 94 94 182 182 182 F F F
103172024-02-27T21:48:00.429Z 16 320-339 94 94 94 182 182 182 F F F
103182024-02-27T21:48:00.429Z 17 340-359 93 93 93 179 179 179 F F F
103192024-02-27T21:48:00.429Z 18 360-379 94 94 94 180 180 180 F F F
103202024-02-27T21:48:00.429Z 19 380-399 93 93 93 179 179 179 F F F
103212024-02-27T21:48:00.429Z 20 400-419 88 88 88 167 167 167 F F F
103222024-02-27T21:48:00.429Z 21 420-439 92 92 92 178 178 178 F F F
103232024-02-27T21:48:00.429Z 22 440-459 92 92 92 178 178 178 F F F
103242024-02-27T21:48:00.430Z 23 460-479 87 87 87 166 166 166 F F F
103252024-02-27T21:48:00.430Z 24 480-499 95 95 95 184 184 184 F F F
103262024-02-27T21:48:00.430Z 25 500-519 94 94 94 181 181 181 F F F
103272024-02-27T21:48:00.430Z 26 520-539 90 90 90 172 172 172 F F F
103282024-02-27T21:48:00.430Z 27 540-559 90 90 90 174 174 174 F F F
103292024-02-27T21:48:00.430Z 28 560-579 86 86 86 161 161 161 F F F
103302024-02-27T21:48:00.431Z 29 580-599 94 94 94 182 182 182 F F F
103312024-02-27T21:48:00.431ZMax gen: 95, Max flush: 184
103322024-02-27T21:48:00.431Z
103332024-02-27T21:48:00.431ZBegin loop 48
103342024-02-27T21:48:00.431ZDownstairs to restart: 1
103352024-02-27T21:48:00.431Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403181683Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528}
103362024-02-27T21:48:00.431Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403440462Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528}
103372024-02-27T21:48:00.432Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.40345868Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528}
103382024-02-27T21:48:00.432Z{"msg":"Crucible e7682e7b-6fce-40a5-8f59-e641d6fd97f6 has session id: 89b71332-4e96-4ee9-88f0-ac6608448aee","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403476728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103392024-02-27T21:48:00.432Z{"msg":"Upstairs opts: Upstairs UUID: e7682e7b-6fce-40a5-8f59-e641d6fd97f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
103402024-02-27T21:48:00.432Z,"time":"2024-02-27T21:47:58.403520573Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103412024-02-27T21:48:00.432Z{"msg":"Crucible stats registered with UUID: e7682e7b-6fce-40a5-8f59-e641d6fd97f6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403562788Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103422024-02-27T21:48:00.432Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403598674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528}
103432024-02-27T21:48:00.433Z{"msg":"e7682e7b-6fce-40a5-8f59-e641d6fd97f6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403763275Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103442024-02-27T21:48:00.433Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403824487Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103452024-02-27T21:48:00.433Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403870892Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103462024-02-27T21:48:00.433Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403908687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103472024-02-27T21:48:00.433Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.403938204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103482024-02-27T21:48:00.433Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.40397001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103492024-02-27T21:48:00.433Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.404016085Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103502024-02-27T21:48:00.434Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.40405619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103512024-02-27T21:48:00.434Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.404089746Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103522024-02-27T21:48:00.434Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.40414255Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103532024-02-27T21:48:00.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.404180465Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103542024-02-27T21:48:00.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.404213831Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103552024-02-27T21:48:00.435Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.404245458Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103562024-02-27T21:48:00.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.404837008Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103572024-02-27T21:48:00.435Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.404873124Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103582024-02-27T21:48:00.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.404890342Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103592024-02-27T21:48:00.435Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.40490442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103602024-02-27T21:48:00.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.405028475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103612024-02-27T21:48:00.435Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.405078929Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103622024-02-27T21:48:00.435Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.405435278Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103632024-02-27T21:48:00.436Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.405644103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103642024-02-27T21:48:00.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.405873036Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103652024-02-27T21:48:00.436Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.405913141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103662024-02-27T21:48:00.436Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.405928469Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103672024-02-27T21:48:00.436Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.405987742Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103682024-02-27T21:48:00.436Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406117837Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103692024-02-27T21:48:00.436Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406161432Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103702024-02-27T21:48:00.437Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406187329Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103712024-02-27T21:48:00.437Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406500282Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103722024-02-27T21:48:00.437Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406541847Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103732024-02-27T21:48:00.437Z{"msg":"[0]R flush_numbers[0..12]: [172, 166, 177, 175, 181, 182, 182, 184, 181, 181, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406558605Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103742024-02-27T21:48:00.437Z{"msg":"[0]R generation[0..12]: [90, 87, 92, 90, 94, 94, 94, 95, 94, 94, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406573004Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103752024-02-27T21:48:00.437Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406588092Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103762024-02-27T21:48:00.437Z{"msg":"[1]R flush_numbers[0..12]: [172, 166, 177, 175, 181, 182, 182, 184, 181, 181, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.40660284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103772024-02-27T21:48:00.438Z{"msg":"[1]R generation[0..12]: [90, 87, 92, 90, 94, 94, 94, 95, 94, 94, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406624497Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103782024-02-27T21:48:00.438Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406640536Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103792024-02-27T21:48:00.438Z{"msg":"[2]R flush_numbers[0..12]: [172, 166, 177, 175, 181, 182, 182, 184, 181, 181, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406661603Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103802024-02-27T21:48:00.438Z{"msg":"[2]R generation[0..12]: [90, 87, 92, 90, 94, 94, 94, 95, 94, 94, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406675851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103812024-02-27T21:48:00.438ZWait for a query_work_queue command to finish before sending IO
103822024-02-27T21:48:00.438Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406701598Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103832024-02-27T21:48:00.438Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406716776Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103842024-02-27T21:48:00.439Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406737314Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103852024-02-27T21:48:00.439Z{"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406750743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103862024-02-27T21:48:00.439Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406764491Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"":"downstairs","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103872024-02-27T21:48:00.439Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406785228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103882024-02-27T21:48:00.439Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406797907Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"0","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103892024-02-27T21:48:00.439Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406812195Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"1","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103902024-02-27T21:48:00.439Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406833313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"client":"2","":"downstairs","session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103912024-02-27T21:48:00.440Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406847481Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103922024-02-27T21:48:00.440Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406866829Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103932024-02-27T21:48:00.440Z{"msg":"e7682e7b-6fce-40a5-8f59-e641d6fd97f6 is now active with session: 89b71332-4e96-4ee9-88f0-ac6608448aee","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406880447Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103942024-02-27T21:48:00.440Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406893226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528,"session_id":"89b71332-4e96-4ee9-88f0-ac6608448aee"}
103952024-02-27T21:48:00.440Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:47:58.406912544Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528}
103962024-02-27T21:48:00.441ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103972024-02-27T21:48:00.441ZLoading write count information from file "/tmp/test_repair_verify.data"
103982024-02-27T21:48:00.441ZRun Repair workload
103992024-02-27T21:48:00.441Z01/30 Write block 296 len 12288 data: 9 9 9
104002024-02-27T21:48:00.441Z02/30 Read block 189 len 12288
104012024-02-27T21:48:00.441Z03/30 Read block 486 len 4096
104022024-02-27T21:48:00.441Z04/30 Flush
104032024-02-27T21:48:00.442Z05/30 Write block 588 len 36864 data: 5 8 8 6 7 7 6 5 6
104042024-02-27T21:48:00.442Z06/30 Read block 398 len 28672
104052024-02-27T21:48:00.442Z07/30 Write block 380 len 20480 data: 4 4 5 5 6
104062024-02-27T21:48:00.442Z08/30 Read block 568 len 12288
104072024-02-27T21:48:00.442Z09/30 Read block 215 len 8192
104082024-02-27T21:48:00.442Z10/30 Read block 452 len 8192
104092024-02-27T21:48:00.449Z11/30 Flush
104102024-02-27T21:48:00.450Z12/30 Flush
104112024-02-27T21:48:00.450Z13/30 Write block 307 len 40960 data: 12 12 12 13 13 12 9 9 7 10
104122024-02-27T21:48:00.450Z14/30 Write block 560 len 24576 data: 11 12 11 10 6 4
104132024-02-27T21:48:00.450Z15/30 Write block 522 len 20480 data: 4 3 3 3 3
104142024-02-27T21:48:00.450Z16/30 Read block 329 len 32768
104152024-02-27T21:48:00.450Z17/30 Read block 243 len 16384
104162024-02-27T21:48:00.450Z18/30 Write block 419 len 20480 data: 8 5 4 4 4
104172024-02-27T21:48:00.451Z19/30 Read block 309 len 28672
104182024-02-27T21:48:00.451Z20/30 Write block 63 len 8192 data: 8 7
104192024-02-27T21:48:00.451Z21/30 Write block 132 len 4096 data: 4
104202024-02-27T21:48:00.451Z22/30 Read block 576 len 12288
104212024-02-27T21:48:00.451Z23/30 Read block 84 len 4096
104222024-02-27T21:48:00.451Z24/30 Read block 167 len 12288
104232024-02-27T21:48:00.451Z25/30 Flush
104242024-02-27T21:48:00.451Z26/30 Flush
104252024-02-27T21:48:00.451Z27/30 Write block 137 len 28672 data: 7 7 9 9 9 10 9
104262024-02-27T21:48:00.452Z28/30 Read block 418 len 24576
104272024-02-27T21:48:00.452Z29/30 Write block 165 len 28672 data: 5 3 3 4 5 6 5
104282024-02-27T21:48:00.452Z30/30 Write block 557 len 20480 data: 15 13 11 12 13
104292024-02-27T21:48:00.452Z----------------------------------------------------------------
104302024-02-27T21:48:00.452Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
104312024-02-27T21:48:00.452ZStates: Active Active Active
104322024-02-27T21:48:00.452ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104332024-02-27T21:48:00.452Z 1 Acked 1000 Write 3 Done Sent Done false
104342024-02-27T21:48:00.453Z 2 Acked 1001 Read 3 Done Sent Done false
104352024-02-27T21:48:00.453Z 3 Acked 1002 Read 1 Done Sent Done false
104362024-02-27T21:48:00.453Z 4 Acked 1003 Flush 0 Done Sent Done false
104372024-02-27T21:48:00.453Z 5 Acked 1004 Write 9 Done Sent Done false
104382024-02-27T21:48:00.453Z 6 Acked 1005 Read 7 Done Sent Done false
104392024-02-27T21:48:00.453Z 7 Acked 1006 Write 5 Done Sent Done false
104402024-02-27T21:48:00.453Z 8 Acked 1007 Read 3 Done Sent Done false
104412024-02-27T21:48:00.453Z 9 Acked 1008 Read 2 Done Sent Done false
104422024-02-27T21:48:00.453Z 10 Acked 1009 Read 2 Done Sent Done false
104432024-02-27T21:48:00.454Z 11 Acked 1010 Flush 0 Done Sent Done false
104442024-02-27T21:48:00.454Z 12 Acked 1011 Flush 0 Done Sent Done false
104452024-02-27T21:48:00.454Z 13 Acked 1012 Write 10 Done Sent Done false
104462024-02-27T21:48:00.454Z 14 Acked 1013 Write 6 Done Sent Done false
104472024-02-27T21:48:00.454Z 15 Acked 1014 Write 5 Done Sent Done false
104482024-02-27T21:48:00.454Z 16 Acked 1015 Read 8 Done Sent Done false
104492024-02-27T21:48:00.454Z 17 Acked 1016 Read 4 Done Sent Done false
104502024-02-27T21:48:00.454Z 18 Acked 1017 Write 5 Done Sent Done false
104512024-02-27T21:48:00.454Z 19 Acked 1018 Read 7 Done Sent Done false
104522024-02-27T21:48:00.454Z 20 Acked 1019 Write 2 Done Sent Done false
104532024-02-27T21:48:00.454Z 21 Acked 1020 Write 1 Done Sent Done false
104542024-02-27T21:48:00.454Z 22 Acked 1021 Read 3 Done Sent Done false
104552024-02-27T21:48:00.455Z 23 Acked 1022 Read 1 Done Sent Done false
104562024-02-27T21:48:00.455Z 24 Acked 1023 Read 3 Done Sent Done false
104572024-02-27T21:48:00.455Z 25 Acked 1024 Flush 0 Done Sent Done false
104582024-02-27T21:48:00.455Z 26 Acked 1025 Flush 0 Done Sent Done false
104592024-02-27T21:48:00.455Z 27 Acked 1026 Write 7 Done Sent Done false
104602024-02-27T21:48:00.455Z 28 Acked 1027 Read 6 Done Sent Done false
104612024-02-27T21:48:00.455Z 29 Acked 1028 Write 7 Done Sent Sent false
104622024-02-27T21:48:00.455Z 30 Acked 1029 Write 5 Sent Sent Sent false
104632024-02-27T21:48:00.456Z STATES DS:0 DS:1 DS:2 TOTAL
104642024-02-27T21:48:00.456Z New 0 0 0 0
104652024-02-27T21:48:00.456Z Sent 1 30 2 33
104662024-02-27T21:48:00.456Z Done 29 0 28 57
104672024-02-27T21:48:00.456Z Skipped 0 0 0 0
104682024-02-27T21:48:00.456Z Error 0 0 0 0
104692024-02-27T21:48:00.456ZLast Flush: 1025 0 1025
104702024-02-27T21:48:00.456ZDownstairs last five completed:
104712024-02-27T21:48:00.456ZUpstairs last five completed: 30 29 28 27 26
104722024-02-27T21:48:00.457Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:47:58.436328839Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1528}
104732024-02-27T21:48:00.457ZWrote out file "/tmp/test_repair_verify.data"
104742024-02-27T21:48:00.457Z
104752024-02-27T21:48:00.457ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104762024-02-27T21:48:00.457ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104772024-02-27T21:48:00.457Z 0 000-019 90 90 90 172 172 172 F F F
104782024-02-27T21:48:00.457Z 1 020-039 87 87 87 166 166 166 F F F
104792024-02-27T21:48:00.457Z 2 040-059 92 92 92 177 177 177 F F F
104802024-02-27T21:48:00.457Z 3 060-079 96 90 96 188 175 188 F F F <---
104812024-02-27T21:48:00.457Z 4 080-099 94 94 94 181 181 181 F F F
104822024-02-27T21:48:00.458Z 5 100-119 94 94 94 182 182 182 F F F
104832024-02-27T21:48:00.458Z 6 120-139 96 94 96 188 182 188 T F T <---
104842024-02-27T21:48:00.458Z 7 140-159 95 95 95 184 184 184 T F T <---
104852024-02-27T21:48:00.458Z 8 160-179 94 94 94 181 181 181 T F T <---
104862024-02-27T21:48:00.458Z 9 180-199 94 94 94 181 181 181 F F F
104872024-02-27T21:48:00.458Z 10 200-219 89 89 89 171 171 171 F F F
104882024-02-27T21:48:00.458Z 11 220-239 89 89 89 171 171 171 F F F
104892024-02-27T21:48:00.458Z 12 240-259 90 90 90 172 172 172 F F F
104902024-02-27T21:48:00.458Z 13 260-279 94 94 94 183 183 183 F F F
104912024-02-27T21:48:00.458Z 14 280-299 96 88 96 185 167 185 F F F <---
104922024-02-27T21:48:00.458Z 15 300-319 96 94 96 188 182 188 F F F <---
104932024-02-27T21:48:00.458Z 16 320-339 94 94 94 182 182 182 F F F
104942024-02-27T21:48:00.459Z 17 340-359 93 93 93 179 179 179 F F F
104952024-02-27T21:48:00.459Z 18 360-379 94 94 94 180 180 180 F F F
104962024-02-27T21:48:00.459Z 19 380-399 96 93 96 186 179 186 F F F <---
104972024-02-27T21:48:00.459Z 20 400-419 96 88 96 188 167 188 F F F <---
104982024-02-27T21:48:00.459Z 21 420-439 96 92 96 188 178 188 F F F <---
104992024-02-27T21:48:00.459Z 22 440-459 92 92 92 178 178 178 F F F
105002024-02-27T21:48:00.459Z 23 460-479 87 87 87 166 166 166 F F F
105012024-02-27T21:48:00.459Z 24 480-499 95 95 95 184 184 184 F F F
105022024-02-27T21:48:00.459Z 25 500-519 94 94 94 181 181 181 F F F
105032024-02-27T21:48:00.459Z 26 520-539 96 90 96 188 172 188 F F F <---
105042024-02-27T21:48:00.459Z 27 540-559 90 90 90 174 174 174 T F T <---
105052024-02-27T21:48:00.459Z 28 560-579 96 86 96 188 161 188 T F T <---
105062024-02-27T21:48:00.459Z 29 580-599 96 94 96 186 182 186 F F F <---
105072024-02-27T21:48:00.459ZMax gen: 96, Max flush: 188
105082024-02-27T21:48:00.459ZError: Difference in extent metadata found!
105092024-02-27T21:48:00.464ZA Difference in extent metadata is expected here
105102024-02-27T21:48:00.464ZOn loop 48
105112024-02-27T21:48:00.465Z
105122024-02-27T21:48:00.465ZVerifying data now
105132024-02-27T21:48:04.728ZNo change after verify
105142024-02-27T21:48:04.728ZLoop: 48 Downstairs dump after verify (and repair):
105152024-02-27T21:48:04.741ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105162024-02-27T21:48:04.741Z 0 000-019 90 90 90 172 172 172 F F F
105172024-02-27T21:48:04.741Z 1 020-039 87 87 87 166 166 166 F F F
105182024-02-27T21:48:04.741Z 2 040-059 92 92 92 177 177 177 F F F
105192024-02-27T21:48:04.741Z 3 060-079 96 96 96 188 188 188 F F F
105202024-02-27T21:48:04.741Z 4 080-099 94 94 94 181 181 181 F F F
105212024-02-27T21:48:04.741Z 5 100-119 94 94 94 182 182 182 F F F
105222024-02-27T21:48:04.741Z 6 120-139 97 97 97 189 189 189 F F F
105232024-02-27T21:48:04.741Z 7 140-159 97 97 97 189 189 189 F F F
105242024-02-27T21:48:04.747Z 8 160-179 97 97 97 189 189 189 F F F
105252024-02-27T21:48:04.747Z 9 180-199 94 94 94 181 181 181 F F F
105262024-02-27T21:48:04.747Z 10 200-219 89 89 89 171 171 171 F F F
105272024-02-27T21:48:04.747Z 11 220-239 89 89 89 171 171 171 F F F
105282024-02-27T21:48:04.747Z 12 240-259 90 90 90 172 172 172 F F F
105292024-02-27T21:48:04.747Z 13 260-279 94 94 94 183 183 183 F F F
105302024-02-27T21:48:04.747Z 14 280-299 96 96 96 185 185 185 F F F
105312024-02-27T21:48:04.748Z 15 300-319 96 96 96 188 188 188 F F F
105322024-02-27T21:48:04.748Z 16 320-339 94 94 94 182 182 182 F F F
105332024-02-27T21:48:04.748Z 17 340-359 93 93 93 179 179 179 F F F
105342024-02-27T21:48:04.748Z 18 360-379 94 94 94 180 180 180 F F F
105352024-02-27T21:48:04.749Z 19 380-399 96 96 96 186 186 186 F F F
105362024-02-27T21:48:04.749Z 20 400-419 96 96 96 188 188 188 F F F
105372024-02-27T21:48:04.749Z 21 420-439 96 96 96 188 188 188 F F F
105382024-02-27T21:48:04.749Z 22 440-459 92 92 92 178 178 178 F F F
105392024-02-27T21:48:04.749Z 23 460-479 87 87 87 166 166 166 F F F
105402024-02-27T21:48:04.749Z 24 480-499 95 95 95 184 184 184 F F F
105412024-02-27T21:48:04.749Z 25 500-519 94 94 94 181 181 181 F F F
105422024-02-27T21:48:04.750Z 26 520-539 96 96 96 188 188 188 F F F
105432024-02-27T21:48:04.750Z 27 540-559 97 97 97 189 189 189 F F F
105442024-02-27T21:48:04.750Z 28 560-579 97 97 97 189 189 189 F F F
105452024-02-27T21:48:04.750Z 29 580-599 96 96 96 186 186 186 F F F
105462024-02-27T21:48:04.750ZMax gen: 97, Max flush: 189
105472024-02-27T21:48:04.750Z
105482024-02-27T21:48:04.750ZBegin loop 49
105492024-02-27T21:48:04.751ZDownstairs to restart: 1
105502024-02-27T21:48:04.762Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.14688743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536}
105512024-02-27T21:48:04.770Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
105522024-02-27T21:48:04.770Z,"time":"2024-02-27T21:48:03.147248498Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536}
105532024-02-27T21:48:04.770Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147302191Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536}
105542024-02-27T21:48:04.771Z{"msg":"Crucible aae2f127-2558-4ee4-811d-5cc5705ddb8d has session id: 14dc2676-f04d-43a0-887e-62a4f8f95a0c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147348016Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105552024-02-27T21:48:04.771Z{"msg":"Upstairs opts: Upstairs UUID: aae2f127-2558-4ee4-811d-5cc5705ddb8d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.14739799Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105562024-02-27T21:48:04.771Z{"msg":"Crucible stats registered with UUID: aae2f127-2558-4ee4-811d-5cc5705ddb8d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147441035Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105572024-02-27T21:48:04.771Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.14747995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536}
105582024-02-27T21:48:04.771Z{"msg":"aae2f127-2558-4ee4-811d-5cc5705ddb8d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147514177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105592024-02-27T21:48:04.771Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147539223Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105602024-02-27T21:48:04.771Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.14756681Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105612024-02-27T21:48:04.772Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147603076Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105622024-02-27T21:48:04.772Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147631503Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105632024-02-27T21:48:04.772Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147670568Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105642024-02-27T21:48:04.772Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147695845Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105652024-02-27T21:48:04.772Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147731291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105662024-02-27T21:48:04.772Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147778085Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105672024-02-27T21:48:04.772Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147801583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105682024-02-27T21:48:04.772Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.14782276Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105692024-02-27T21:48:04.773Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.147974342Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105702024-02-27T21:48:04.773Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.148071901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105712024-02-27T21:48:04.773Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.148485162Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105722024-02-27T21:48:04.773Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.148521828Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105732024-02-27T21:48:04.773Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.148610047Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105742024-02-27T21:48:04.773Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.148657272Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105752024-02-27T21:48:04.773Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.148694377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105762024-02-27T21:48:04.774Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.148725884Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105772024-02-27T21:48:04.774Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.148994622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105782024-02-27T21:48:04.774Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149118328Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105792024-02-27T21:48:04.774Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149363579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105802024-02-27T21:48:04.774Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149496393Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105812024-02-27T21:48:04.774Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149547357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105822024-02-27T21:48:04.774Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149587922Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105832024-02-27T21:48:04.774Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149637256Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105842024-02-27T21:48:04.774Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149669052Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105852024-02-27T21:48:04.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149684591Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105862024-02-27T21:48:04.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.14986201Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105872024-02-27T21:48:04.775Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149903865Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105882024-02-27T21:48:04.775Z{"msg":"[0]R flush_numbers[0..12]: [172, 166, 177, 188, 181, 182, 189, 189, 189, 181, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149930532Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105892024-02-27T21:48:04.775Z{"msg":"[0]R generation[0..12]: [90, 87, 92, 96, 94, 94, 97, 97, 97, 94, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.149966088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105902024-02-27T21:48:04.775Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150001233Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105912024-02-27T21:48:04.775Z{"msg":"[1]R flush_numbers[0..12]: [172, 166, 177, 188, 181, 182, 189, 189, 189, 181, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150038319Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105922024-02-27T21:48:04.775Z{"msg":"[1]R generation[0..12]: [90, 87, 92, 96, 94, 94, 97, 97, 97, 94, 89, 89]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
105932024-02-27T21:48:04.775Z,"time":"2024-02-27T21:48:03.150075795Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105942024-02-27T21:48:04.776Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.15011721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105952024-02-27T21:48:04.776Z{"msg":"[2]R flush_numbers[0..12]: [172, 166, 177, 188, 181, 182, 189, 189, 189, 181, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150155245Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105962024-02-27T21:48:04.776Z{"msg":"[2]R generation[0..12]: [90, 87, 92, 96, 94, 94, 97, 97, 97, 94, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.15020193Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105972024-02-27T21:48:04.776Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150243135Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105982024-02-27T21:48:04.776Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150279241Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
105992024-02-27T21:48:04.776Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150308157Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106002024-02-27T21:48:04.776Z{"msg"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
106012024-02-27T21:48:04.776Z:"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150349832Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106022024-02-27T21:48:04.777Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150380529Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"":"downstairs","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106032024-02-27T21:48:04.777Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150397317Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106042024-02-27T21:48:04.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150416435Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"0","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106052024-02-27T21:48:04.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150430153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"1","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106062024-02-27T21:48:04.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.15045408Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"client":"2","":"downstairs","session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106072024-02-27T21:48:04.777Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150478617Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106082024-02-27T21:48:04.777Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150491976Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106092024-02-27T21:48:04.777Z{"msg":"aae2f127-2558-4ee4-811d-5cc5705ddb8d is now active with session: 14dc2676-f04d-43a0-887e-62a4f8f95a0c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150512533Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106102024-02-27T21:48:04.777Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.150524942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536,"session_id":"14dc2676-f04d-43a0-887e-62a4f8f95a0c"}
106112024-02-27T21:48:04.777Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:03.15054484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536}
106122024-02-27T21:48:04.777ZLoading write count information from file "/tmp/test_repair_verify.data"
106132024-02-27T21:48:04.778ZRun Repair workload
106142024-02-27T21:48:04.782Z01/30 Write block 45 len 36864 data: 12 8 8 8 7 8 7 8 8
106152024-02-27T21:48:04.783Z02/30 Read block 548 len 40960
106162024-02-27T21:48:04.783Z03/30 Read block 169 len 28672
106172024-02-27T21:48:04.783Z04/30 Write block 244 len 28672 data: 6 7 8 8 7 7 7
106182024-02-27T21:48:04.783Z05/30 Read block 477 len 32768
106192024-02-27T21:48:04.783Z06/30 Read block 335 len 8192
106202024-02-27T21:48:04.783Z07/30 Write block 93 len 16384 data: 6 7 6 6
106212024-02-27T21:48:04.783Z08/30 Read block 240 len 28672
106222024-02-27T21:48:04.783Z09/30 Read block 327 len 20480
106232024-02-27T21:48:04.783Z10/30 Read block 197 len 36864
106242024-02-27T21:48:04.783Z11/30 Write block 430 len 36864 data: 9 9 9 11 13 10 7 7 6
106252024-02-27T21:48:04.784Z12/30 Read block 515 len 32768
106262024-02-27T21:48:04.784Z13/30 Write block 391 len 40960 data: 10 8 9 7 7 7 10 9 9 10
106272024-02-27T21:48:04.784Z14/30 Write block 77 len 4096 data: 6
106282024-02-27T21:48:04.784Z15/30 Write block 349 len 12288 data: 9 9 9
106292024-02-27T21:48:04.784Z16/30 Write block 83 len 36864 data: 4 5 5 5 6 8 9 7 8
106302024-02-27T21:48:04.784Z17/30 Read block 579 len 12288
106312024-02-27T21:48:04.790Z18/30 Write block 184 len 24576 data: 12 8 9 9 10 10
106322024-02-27T21:48:04.790Z19/30 Read block 90 len 24576
106332024-02-27T21:48:04.790Z20/30 Write block 365 len 16384 data: 9 9 9 9
106342024-02-27T21:48:04.790Z21/30 Read block 13 len 12288
106352024-02-27T21:48:04.790Z22/30 Write block 335 len 16384 data: 11 8 9 10
106362024-02-27T21:48:04.790Z23/30 Read block 434 len 24576
106372024-02-27T21:48:04.790Z24/30 Read block 211 len 8192
106382024-02-27T21:48:04.790Z25/30 Write block 92 len 24576 data: 4 7 8 7 7 6
106392024-02-27T21:48:04.791Z26/30 Read block 16 len 24576
106402024-02-27T21:48:04.791Z27/30 Read block 87 len 36864
106412024-02-27T21:48:04.795Z28/30 Read block 140 len 12288
106422024-02-27T21:48:04.795Z29/30 Write block 511 len 28672 data: 15 14 12 10 8 9 9
106432024-02-27T21:48:04.795Z30/30 Read block 369 len 8192
106442024-02-27T21:48:04.796Z----------------------------------------------------------------
106452024-02-27T21:48:04.796Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
106462024-02-27T21:48:04.796ZStates: Active Active Active
106472024-02-27T21:48:04.796ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106482024-02-27T21:48:04.796Z 1 Acked 1000 Write 9 Done Done Done false
106492024-02-27T21:48:04.796Z 2 Acked 1001 Read 10 Done Done Done false
106502024-02-27T21:48:04.796Z 3 Acked 1002 Read 7 Done Done Done false
106512024-02-27T21:48:04.796Z 4 Acked 1003 Write 7 Done Done Done false
106522024-02-27T21:48:04.796Z 5 Acked 1004 Read 8 Done Sent Done false
106532024-02-27T21:48:04.796Z 6 Acked 1005 Read 2 Done Sent Done false
106542024-02-27T21:48:04.796Z 7 Acked 1006 Write 4 Done Sent Done false
106552024-02-27T21:48:04.797Z 8 Acked 1007 Read 7 Done Sent Done false
106562024-02-27T21:48:04.797Z 9 Acked 1008 Read 5 Done Sent Done false
106572024-02-27T21:48:04.797Z 10 Acked 1009 Read 9 Done Sent Done false
106582024-02-27T21:48:04.797Z 11 Acked 1010 Write 9 Done Sent Done false
106592024-02-27T21:48:04.797Z 12 Acked 1011 Read 8 Done Sent Done false
106602024-02-27T21:48:04.797Z 13 Acked 1012 Write 10 Done Sent Done false
106612024-02-27T21:48:04.797Z 14 Acked 1013 Write 1 Done Sent Done false
106622024-02-27T21:48:04.797Z 15 Acked 1014 Write 3 Done Sent Done false
106632024-02-27T21:48:04.797Z 16 Acked 1015 Write 9 Done Sent Done false
106642024-02-27T21:48:04.797Z 17 Acked 1016 Read 3 Done Sent Done false
106652024-02-27T21:48:04.798Z 18 Acked 1017 Write 6 Done Sent Done false
106662024-02-27T21:48:04.798Z 19 Acked 1018 Read 6 Done Sent Done false
106672024-02-27T21:48:04.798Z 20 Acked 1019 Write 4 Done Sent Done false
106682024-02-27T21:48:04.798Z 21 Acked 1020 Read 3 Done Sent Done false
106692024-02-27T21:48:04.798Z 22 Acked 1021 Write 4 Done Sent Done false
106702024-02-27T21:48:04.798Z 23 Acked 1022 Read 6 Done Sent Done false
106712024-02-27T21:48:04.798Z 24 Acked 1023 Read 2 Done Sent Done false
106722024-02-27T21:48:04.798Z 25 Acked 1024 Write 6 Done Sent Done false
106732024-02-27T21:48:04.798Z 26 Acked 1025 Read 6 Done Sent Done false
106742024-02-27T21:48:04.798Z 27 Acked 1026 Read 9 Done Sent Done false
106752024-02-27T21:48:04.798Z 28 Acked 1027 Read 3 Done Sent Done false
106762024-02-27T21:48:04.798Z 29 Acked 1028 Write 7 Done Sent Done false
106772024-02-27T21:48:04.799Z 30 Acked 1029 Read 2 Done Sent Done false
106782024-02-27T21:48:04.799Z STATES DS:0 DS:1 DS:2 TOTAL
106792024-02-27T21:48:04.799Z New 0 0 0 0
106802024-02-27T21:48:04.799Z Sent 0 26 0 26
106812024-02-27T21:48:04.799Z Done 30 4 30 64
106822024-02-27T21:48:04.799Z Skipped 0 0 0 0
106832024-02-27T21:48:04.799Z Error 0 0 0 0
106842024-02-27T21:48:04.799ZLast Flush: 0 0 0
106852024-02-27T21:48:04.799ZDownstairs last five completed:
106862024-02-27T21:48:04.799ZUpstairs last five completed: 30 29 28 27 26
106872024-02-27T21:48:04.799Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:03.178263914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1536}
106882024-02-27T21:48:04.799ZWrote out file "/tmp/test_repair_verify.data"
106892024-02-27T21:48:04.803Z
106902024-02-27T21:48:04.803ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106912024-02-27T21:48:04.821ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106922024-02-27T21:48:04.821Z 0 000-019 90 90 90 172 172 172 F F F
106932024-02-27T21:48:04.821Z 1 020-039 87 87 87 166 166 166 F F F
106942024-02-27T21:48:04.821Z 2 040-059 92 92 92 177 177 177 T T T
106952024-02-27T21:48:04.822Z 3 060-079 96 96 96 188 188 188 T F T <---
106962024-02-27T21:48:04.822Z 4 080-099 94 94 94 181 181 181 T F T <---
106972024-02-27T21:48:04.822Z 5 100-119 94 94 94 182 182 182 F F F
106982024-02-27T21:48:04.822Z 6 120-139 97 97 97 189 189 189 F F F
106992024-02-27T21:48:04.822Z 7 140-159 97 97 97 189 189 189 F F F
107002024-02-27T21:48:04.822Z 8 160-179 97 97 97 189 189 189 F F F
107012024-02-27T21:48:04.823Z 9 180-199 94 94 94 181 181 181 T F T <---
107022024-02-27T21:48:04.823Z 10 200-219 89 89 89 171 171 171 F F F
107032024-02-27T21:48:04.823Z 11 220-239 89 89 89 171 171 171 F F F
107042024-02-27T21:48:04.823Z 12 240-259 90 90 90 172 172 172 T T T
107052024-02-27T21:48:04.823Z 13 260-279 94 94 94 183 183 183 F F F
107062024-02-27T21:48:04.823Z 14 280-299 96 96 96 185 185 185 F F F
107072024-02-27T21:48:04.823Z 15 300-319 96 96 96 188 188 188 F F F
107082024-02-27T21:48:04.823Z 16 320-339 94 94 94 182 182 182 T F T <---
107092024-02-27T21:48:04.823Z 17 340-359 93 93 93 179 179 179 T F T <---
107102024-02-27T21:48:04.823Z 18 360-379 94 94 94 180 180 180 T F T <---
107112024-02-27T21:48:04.824Z 19 380-399 96 96 96 186 186 186 T F T <---
107122024-02-27T21:48:04.824Z 20 400-419 96 96 96 188 188 188 T F T <---
107132024-02-27T21:48:04.830Z 21 420-439 96 96 96 188 188 188 T F T <---
107142024-02-27T21:48:04.830Z 22 440-459 92 92 92 178 178 178 F F F
107152024-02-27T21:48:04.830Z 23 460-479 87 87 87 166 166 166 F F F
107162024-02-27T21:48:04.831Z 24 480-499 95 95 95 184 184 184 F F F
107172024-02-27T21:48:04.831Z 25 500-519 94 94 94 181 181 181 T F T <---
107182024-02-27T21:48:04.831Z 26 520-539 96 96 96 188 188 188 F F F
107192024-02-27T21:48:04.831Z 27 540-559 97 97 97 189 189 189 F F F
107202024-02-27T21:48:04.831Z 28 560-579 97 97 97 189 189 189 F F F
107212024-02-27T21:48:04.831Z 29 580-599 96 96 96 186 186 186 F F F
107222024-02-27T21:48:04.831ZMax gen: 97, Max flush: 189
107232024-02-27T21:48:04.831ZError: Difference in extent metadata found!
107242024-02-27T21:48:04.831ZA Difference in extent metadata is expected here
107252024-02-27T21:48:04.831ZOn loop 49
107262024-02-27T21:48:04.832Z
107272024-02-27T21:48:04.832ZVerifying data now
107282024-02-27T21:48:10.871ZNo change after verify
107292024-02-27T21:48:10.871ZLoop: 49 Downstairs dump after verify (and repair):
107302024-02-27T21:48:10.893ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107312024-02-27T21:48:10.893Z 0 000-019 90 90 90 172 172 172 F F F
107322024-02-27T21:48:10.894Z 1 020-039 87 87 87 166 166 166 F F F
107332024-02-27T21:48:10.894Z 2 040-059 98 98 98 190 190 190 F F F
107342024-02-27T21:48:10.894Z 3 060-079 98 98 98 190 190 190 F F F
107352024-02-27T21:48:10.894Z 4 080-099 98 98 98 190 190 190 F F F
107362024-02-27T21:48:10.894Z 5 100-119 94 94 94 182 182 182 F F F
107372024-02-27T21:48:10.894Z 6 120-139 97 97 97 189 189 189 F F F
107382024-02-27T21:48:10.895Z 7 140-159 97 97 97 189 189 189 F F F
107392024-02-27T21:48:10.895Z 8 160-179 97 97 97 189 189 189 F F F
107402024-02-27T21:48:10.895Z 9 180-199 98 98 98 190 190 190 F F F
107412024-02-27T21:48:10.895Z 10 200-219 89 89 89 171 171 171 F F F
107422024-02-27T21:48:10.895Z 11 220-239 89 89 89 171 171 171 F F F
107432024-02-27T21:48:10.895Z 12 240-259 98 98 98 190 190 190 F F F
107442024-02-27T21:48:10.895Z 13 260-279 94 94 94 183 183 183 F F F
107452024-02-27T21:48:10.895Z 14 280-299 96 96 96 185 185 185 F F F
107462024-02-27T21:48:10.896Z 15 300-319 96 96 96 188 188 188 F F F
107472024-02-27T21:48:10.896Z 16 320-339 98 98 98 190 190 190 F F F
107482024-02-27T21:48:10.896Z 17 340-359 98 98 98 190 190 190 F F F
107492024-02-27T21:48:10.896Z 18 360-379 98 98 98 190 190 190 F F F
107502024-02-27T21:48:10.896Z 19 380-399 98 98 98 190 190 190 F F F
107512024-02-27T21:48:10.897Z 20 400-419 98 98 98 190 190 190 F F F
107522024-02-27T21:48:10.897Z 21 420-439 98 98 98 190 190 190 F F F
107532024-02-27T21:48:10.897Z 22 440-459 92 92 92 178 178 178 F F F
107542024-02-27T21:48:10.897Z 23 460-479 87 87 87 166 166 166 F F F
107552024-02-27T21:48:10.897Z 24 480-499 95 95 95 184 184 184 F F F
107562024-02-27T21:48:10.897Z 25 500-519 98 98 98 190 190 190 F F F
107572024-02-27T21:48:10.898Z 26 520-539 96 96 96 188 188 188 F F F
107582024-02-27T21:48:10.898Z 27 540-559 97 97 97 189 189 189 F F F
107592024-02-27T21:48:10.898Z 28 560-579 97 97 97 189 189 189 F F F
107602024-02-27T21:48:10.898Z 29 580-599 96 96 96 186 186 186 F F F
107612024-02-27T21:48:10.899ZMax gen: 98, Max flush: 190
107622024-02-27T21:48:10.899Z
107632024-02-27T21:48:10.899ZBegin loop 50
107642024-02-27T21:48:10.899ZDownstairs to restart: 2
107652024-02-27T21:48:10.899Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.877423187Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544}
107662024-02-27T21:48:10.899Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.877693865Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544}
107672024-02-27T21:48:10.900Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.877721542Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544}
107682024-02-27T21:48:10.900Z{"msg":"Crucible 27376cad-d8ad-4c5b-a311-02d918e39492 has session id: 73f37eed-14ad-4e84-a38d-02baf169fcd7","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.877761467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544Crucible runtime is spawned
107692024-02-27T21:48:10.900Z,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107702024-02-27T21:48:10.900Z{"msg":"Upstairs opts: Upstairs UUID: 27376cad-d8ad-4c5b-a311-02d918e39492, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.877814121Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107712024-02-27T21:48:10.901Z{"msg":"Crucible stats registered with UUID: 27376cad-d8ad-4c5b-a311-02d918e39492","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.87790867Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107722024-02-27T21:48:10.901Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878016647Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544}
107732024-02-27T21:48:10.901Z{"msg":"27376cad-d8ad-4c5b-a311-02d918e39492 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878110476Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107742024-02-27T21:48:10.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878153711Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107752024-02-27T21:48:10.902Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878200986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107762024-02-27T21:48:10.902Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.87824253Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107772024-02-27T21:48:10.902Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878268358Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107782024-02-27T21:48:10.903Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878354037Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107792024-02-27T21:48:10.903Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878394703Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107802024-02-27T21:48:10.903Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878424409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107812024-02-27T21:48:10.903Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878459625Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107822024-02-27T21:48:10.903Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.8784992Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107832024-02-27T21:48:10.904Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878529637Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107842024-02-27T21:48:10.904Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878580391Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107852024-02-27T21:48:10.904Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.878634244Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107862024-02-27T21:48:10.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.879227965Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107872024-02-27T21:48:10.904Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.87926667Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107882024-02-27T21:48:10.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.879283288Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107892024-02-27T21:48:10.905Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.879297596Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107902024-02-27T21:48:10.905Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.879311975Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107912024-02-27T21:48:10.905Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.879375567Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107922024-02-27T21:48:10.905Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.879893166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107932024-02-27T21:48:10.905Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.87994179Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107942024-02-27T21:48:10.906Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.879989045Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107952024-02-27T21:48:10.906Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880426223Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107962024-02-27T21:48:10.906Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880478707Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107972024-02-27T21:48:10.906Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880531721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107982024-02-27T21:48:10.906Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880573436Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
107992024-02-27T21:48:10.907Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880609512Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108002024-02-27T21:48:10.907Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880642898Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108012024-02-27T21:48:10.907Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880675244Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108022024-02-27T21:48:10.907Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.8807086Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108032024-02-27T21:48:10.907Z{"msg":"[0]R flush_numbers[0..12]: [172, 166, 190, 190, 190, 182, 189, 189, 189, 190, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880744326Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108042024-02-27T21:48:10.907Z{"msg":"Wait for a query_work_queue command to finish before sending IO
108052024-02-27T21:48:10.907Z[0]R generation[0..12]: [90, 87, 98, 98, 98, 94, 97, 97, 97, 98, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880785871Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108062024-02-27T21:48:10.907Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880827296Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108072024-02-27T21:48:10.908Z{"msg":"[1]R flush_numbers[0..12]: [172, 166, 190, 190, 190, 182, 189, 189, 189, 190, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880856843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108082024-02-27T21:48:10.908Z{"msg":"[1]R generation[0..12]: [90, 87, 98, 98, 98, 94, 97, 97, 97, 98, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880892418Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108092024-02-27T21:48:10.908Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880928694Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108102024-02-27T21:48:10.908Z{"msg":"[2]R flush_numbers[0..12]: [172, 166, 190, 190, 190, 182, 189, 189, 189, 190, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880955381Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108112024-02-27T21:48:10.908Z{"msg":"[2]R generation[0..12]: [90, 87, 98, 98, 98, 94, 97, 97, 97, 98, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880971109Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108122024-02-27T21:48:10.908Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.880986297Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108132024-02-27T21:48:10.908Z{"msg":"Max found gen is 99","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881023513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108142024-02-27T21:48:10.908Z{"msg":"Generation requested: 100 >= found:99","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881039561Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108152024-02-27T21:48:10.909Z{"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881062878Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108162024-02-27T21:48:10.909Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881080896Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"":"downstairs","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108172024-02-27T21:48:10.909Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881106453Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108182024-02-27T21:48:10.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881121231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"0","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108192024-02-27T21:48:10.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881142289Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"1","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108202024-02-27T21:48:10.910Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881156147Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"client":"2","":"downstairs","session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108212024-02-27T21:48:10.910Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881176805Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108222024-02-27T21:48:10.910Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881190343Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108232024-02-27T21:48:10.910Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108242024-02-27T21:48:10.910Z27376cad-d8ad-4c5b-a311-02d918e39492 is now active with session: 73f37eed-14ad-4e84-a38d-02baf169fcd7","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.88121962Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108252024-02-27T21:48:10.910Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881238668Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544,"session_id":"73f37eed-14ad-4e84-a38d-02baf169fcd7"}
108262024-02-27T21:48:10.910Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:08.881251446Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544}
108272024-02-27T21:48:10.911ZLoading write count information from file "/tmp/test_repair_verify.data"
108282024-02-27T21:48:10.911ZRun Repair workload
108292024-02-27T21:48:10.911Z01/30 Write block 110 len 4096 data: 10
108302024-02-27T21:48:10.934Z02/30 Read block 99 len 28672
108312024-02-27T21:48:10.935Z03/30 Read block 403 len 40960
108322024-02-27T21:48:10.935Z04/30 Write block 478 len 24576 data: 6 6 8 9 11 13
108332024-02-27T21:48:10.935Z05/30 Write block 33 len 4096 data: 9
108342024-02-27T21:48:10.936Z06/30 Read block 361 len 4096
108352024-02-27T21:48:10.936Z07/30 Write block 588 len 8192 data: 6 9
108362024-02-27T21:48:10.936Z08/30 Write block 381 len 16384 data: 5 6 6 7
108372024-02-27T21:48:10.936Z09/30 Read block 361 len 36864
108382024-02-27T21:48:10.936Z10/30 Read block 172 len 32768
108392024-02-27T21:48:10.937Z11/30 Read block 35 len 24576
108402024-02-27T21:48:10.937Z12/30 Write block 133 len 16384 data: 5 4 6 6
108412024-02-27T21:48:10.937Z13/30 Read block 524 len 20480
108422024-02-27T21:48:10.937Z14/30 Read block 134 len 36864
108432024-02-27T21:48:10.937Z15/30 Read block 249 len 24576
108442024-02-27T21:48:10.938Z16/30 Read block 464 len 16384
108452024-02-27T21:48:10.938Z17/30 Read block 425 len 24576
108462024-02-27T21:48:10.938Z18/30 Write block 440 len 16384 data: 7 5 5 6
108472024-02-27T21:48:10.938Z19/30 Read block 452 len 24576
108482024-02-27T21:48:10.939Z20/30 Read block 308 len 20480
108492024-02-27T21:48:10.939Z21/30 Write block 338 len 16384 data: 11 10 11 11
108502024-02-27T21:48:10.939Z22/30 Write block 476 len 16384 data: 7 6 7 7
108512024-02-27T21:48:10.939Z23/30 Read block 189 len 16384
108522024-02-27T21:48:10.939Z24/30 Read block 447 len 16384
108532024-02-27T21:48:10.939Z25/30 Write block 399 len 32768 data: 10 11 10 7 6 7 6 6
108542024-02-27T21:48:10.939Z26/30 Read block 253 len 40960
108552024-02-27T21:48:10.940Z27/30 Write block 502 len 20480 data: 4 4 6 7 8
108562024-02-27T21:48:10.940Z28/30 Read block 547 len 28672
108572024-02-27T21:48:10.940Z29/30 Write block 472 len 20480 data: 5 6 5 7 8
108582024-02-27T21:48:10.940Z30/30 Write block 362 len 8192 data: 4 8
108592024-02-27T21:48:10.940Z----------------------------------------------------------------
108602024-02-27T21:48:10.940Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
108612024-02-27T21:48:10.940ZStates: Active Active Active
108622024-02-27T21:48:10.940ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108632024-02-27T21:48:10.941Z 1 Acked 1000 Write 1 Done Done Done false
108642024-02-27T21:48:10.941Z 2 Acked 1001 Read 7 Done Done Done false
108652024-02-27T21:48:10.941Z 3 Acked 1002 Read 10 Done Done Done false
108662024-02-27T21:48:10.941Z 4 Acked 1003 Write 6 Done Done Done false
108672024-02-27T21:48:10.941Z 5 Acked 1004 Write 1 Done Done Done false
108682024-02-27T21:48:10.941Z 6 Acked 1005 Read 1 Done Done Sent false
108692024-02-27T21:48:10.941Z 7 Acked 1006 Write 2 Done Done Sent false
108702024-02-27T21:48:10.941Z 8 Acked 1007 Write 4 Done Done Sent false
108712024-02-27T21:48:10.942Z 9 Acked 1008 Read 9 Done Done Sent false
108722024-02-27T21:48:10.942Z 10 Acked 1009 Read 8 Done Done Sent false
108732024-02-27T21:48:10.942Z 11 Acked 1010 Read 6 Done Done Sent false
108742024-02-27T21:48:10.942Z 12 Acked 1011 Write 4 Done Done Sent false
108752024-02-27T21:48:10.942Z 13 Acked 1012 Read 5 Done Done Sent false
108762024-02-27T21:48:10.942Z 14 Acked 1013 Read 9 Done Done Sent false
108772024-02-27T21:48:10.942Z 15 Acked 1014 Read 6 Done Done Sent false
108782024-02-27T21:48:10.942Z 16 Acked 1015 Read 4 Done Done Sent false
108792024-02-27T21:48:10.942Z 17 Acked 1016 Read 6 Done Done Sent false
108802024-02-27T21:48:10.942Z 18 Acked 1017 Write 4 Done Done Sent false
108812024-02-27T21:48:10.942Z 19 Acked 1018 Read 6 Done Done Sent false
108822024-02-27T21:48:10.943Z 20 Acked 1019 Read 5 Done Done Sent false
108832024-02-27T21:48:10.943Z 21 Acked 1020 Write 4 Done Done Sent false
108842024-02-27T21:48:10.943Z 22 Acked 1021 Write 4 Done Done Sent false
108852024-02-27T21:48:10.943Z 23 Acked 1022 Read 4 Done Done Sent false
108862024-02-27T21:48:10.943Z 24 Acked 1023 Read 4 Done Done Sent false
108872024-02-27T21:48:10.943Z 25 Acked 1024 Write 8 Done Done Sent false
108882024-02-27T21:48:10.943Z 26 Acked 1025 Read 10 Done Done Sent false
108892024-02-27T21:48:10.943Z 27 Acked 1026 Write 5 Done Done Sent false
108902024-02-27T21:48:10.943Z 28 Acked 1027 Read 7 Done Done Sent false
108912024-02-27T21:48:10.943Z 29 Acked 1028 Write 5 Sent Sent Sent false
108922024-02-27T21:48:10.944Z 30 Acked 1029 Write 2 Sent Sent Sent false
108932024-02-27T21:48:10.944Z STATES DS:0 DS:1 DS:2 TOTAL
108942024-02-27T21:48:10.944Z New 0 0 0 0
108952024-02-27T21:48:10.944Z Sent 2 2 25 29
108962024-02-27T21:48:10.944Z Done 28 28 5 61
108972024-02-27T21:48:10.944Z Skipped 0 0 0 0
108982024-02-27T21:48:10.944Z Error 0 0 0 0
108992024-02-27T21:48:10.945ZLast Flush: 0 0 0
109002024-02-27T21:48:10.945ZDownstairs last five completed:
109012024-02-27T21:48:10.945ZUpstairs last five completed: 30 29 28 27 26
109022024-02-27T21:48:10.945Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:08.907443501Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1544}
109032024-02-27T21:48:10.945ZWrote out file "/tmp/test_repair_verify.data"
109042024-02-27T21:48:10.945Z
109052024-02-27T21:48:10.945ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
109062024-02-27T21:48:10.946ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109072024-02-27T21:48:10.946Z 0 000-019 90 90 90 172 172 172 F F F
109082024-02-27T21:48:10.946Z 1 020-039 87 87 87 166 166 166 T T T
109092024-02-27T21:48:10.946Z 2 040-059 98 98 98 190 190 190 F F F
109102024-02-27T21:48:10.946Z 3 060-079 98 98 98 190 190 190 F F F
109112024-02-27T21:48:10.947Z 4 080-099 98 98 98 190 190 190 F F F
109122024-02-27T21:48:10.947Z 5 100-119 94 94 94 182 182 182 T T T
109132024-02-27T21:48:10.947Z 6 120-139 97 97 97 189 189 189 T T F <---
109142024-02-27T21:48:10.947Z 7 140-159 97 97 97 189 189 189 F F F
109152024-02-27T21:48:10.947Z 8 160-179 97 97 97 189 189 189 F F F
109162024-02-27T21:48:10.947Z 9 180-199 98 98 98 190 190 190 F F F
109172024-02-27T21:48:10.947Z 10 200-219 89 89 89 171 171 171 F F F
109182024-02-27T21:48:10.947Z 11 220-239 89 89 89 171 171 171 F F F
109192024-02-27T21:48:10.947Z 12 240-259 98 98 98 190 190 190 F F F
109202024-02-27T21:48:10.948Z 13 260-279 94 94 94 183 183 183 F F F
109212024-02-27T21:48:10.948Z 14 280-299 96 96 96 185 185 185 F F F
109222024-02-27T21:48:10.948Z 15 300-319 96 96 96 188 188 188 F F F
109232024-02-27T21:48:10.948Z 16 320-339 98 98 98 190 190 190 T T F <---
109242024-02-27T21:48:10.948Z 17 340-359 98 98 98 190 190 190 T T F <---
109252024-02-27T21:48:10.948Z 18 360-379 98 98 98 190 190 190 T T F <---
109262024-02-27T21:48:10.949Z 19 380-399 98 98 98 190 190 190 T T F <---
109272024-02-27T21:48:10.949Z 20 400-419 98 98 98 190 190 190 T T F <---
109282024-02-27T21:48:10.949Z 21 420-439 98 98 98 190 190 190 F F F
109292024-02-27T21:48:10.949Z 22 440-459 92 92 92 178 178 178 T T F <---
109302024-02-27T21:48:10.956Z 23 460-479 87 87 87 166 166 166 T T T
109312024-02-27T21:48:10.956Z 24 480-499 95 95 95 184 184 184 T T T
109322024-02-27T21:48:10.956Z 25 500-519 98 98 98 190 190 190 T T F <---
109332024-02-27T21:48:10.956Z 26 520-539 96 96 96 188 188 188 F F F
109342024-02-27T21:48:10.956Z 27 540-559 97 97 97 189 189 189 F F F
109352024-02-27T21:48:10.957Z 28 560-579 97 97 97 189 189 189 F F F
109362024-02-27T21:48:10.957Z 29 580-599 96 96 96 186 186 186 T T F <---
109372024-02-27T21:48:10.957ZMax gen: 98, Max flush: 190
109382024-02-27T21:48:10.957ZError: Difference in extent metadata found!
109392024-02-27T21:48:10.957ZA Difference in extent metadata is expected here
109402024-02-27T21:48:10.957ZOn loop 50
109412024-02-27T21:48:10.957Z
109422024-02-27T21:48:10.957ZVerifying data now
109432024-02-27T21:48:16.230ZNo change after verify
109442024-02-27T21:48:16.230ZLoop: 50 Downstairs dump after verify (and repair):
109452024-02-27T21:48:16.244ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109462024-02-27T21:48:16.244Z 0 000-019 90 90 90 172 172 172 F F F
109472024-02-27T21:48:16.245Z 1 020-039 99 99 99 191 191 191 F F F
109482024-02-27T21:48:16.245Z 2 040-059 98 98 98 190 190 190 F F F
109492024-02-27T21:48:16.249Z 3 060-079 98 98 98 190 190 190 F F F
109502024-02-27T21:48:16.249Z 4 080-099 98 98 98 190 190 190 F F F
109512024-02-27T21:48:16.249Z 5 100-119 99 99 99 191 191 191 F F F
109522024-02-27T21:48:16.249Z 6 120-139 99 99 99 191 191 191 F F F
109532024-02-27T21:48:16.249Z 7 140-159 97 97 97 189 189 189 F F F
109542024-02-27T21:48:16.250Z 8 160-179 97 97 97 189 189 189 F F F
109552024-02-27T21:48:16.250Z 9 180-199 98 98 98 190 190 190 F F F
109562024-02-27T21:48:16.250Z 10 200-219 89 89 89 171 171 171 F F F
109572024-02-27T21:48:16.250Z 11 220-239 89 89 89 171 171 171 F F F
109582024-02-27T21:48:16.250Z 12 240-259 98 98 98 190 190 190 F F F
109592024-02-27T21:48:16.250Z 13 260-279 94 94 94 183 183 183 F F F
109602024-02-27T21:48:16.251Z 14 280-299 96 96 96 185 185 185 F F F
109612024-02-27T21:48:16.251Z 15 300-319 96 96 96 188 188 188 F F F
109622024-02-27T21:48:16.251Z 16 320-339 99 99 99 191 191 191 F F F
109632024-02-27T21:48:16.251Z 17 340-359 99 99 99 191 191 191 F F F
109642024-02-27T21:48:16.251Z 18 360-379 99 99 99 191 191 191 F F F
109652024-02-27T21:48:16.252Z 19 380-399 99 99 99 191 191 191 F F F
109662024-02-27T21:48:16.252Z 20 400-419 99 99 99 191 191 191 F F F
109672024-02-27T21:48:16.252Z 21 420-439 98 98 98 190 190 190 F F F
109682024-02-27T21:48:16.252Z 22 440-459 99 99 99 191 191 191 F F F
109692024-02-27T21:48:16.252Z 23 460-479 99 99 99 191 191 191 F F F
109702024-02-27T21:48:16.252Z 24 480-499 99 99 99 191 191 191 F F F
109712024-02-27T21:48:16.252Z 25 500-519 99 99 99 191 191 191 F F F
109722024-02-27T21:48:16.252Z 26 520-539 96 96 96 188 188 188 F F F
109732024-02-27T21:48:16.253Z 27 540-559 97 97 97 189 189 189 F F F
109742024-02-27T21:48:16.253Z 28 560-579 97 97 97 189 189 189 F F F
109752024-02-27T21:48:16.253Z 29 580-599 99 99 99 191 191 191 F F F
109762024-02-27T21:48:16.253ZMax gen: 99, Max flush: 191
109772024-02-27T21:48:16.253Z
109782024-02-27T21:48:16.253ZBegin loop 51
109792024-02-27T21:48:16.254ZDownstairs to restart: 1
109802024-02-27T21:48:16.267Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650234905Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552}
109812024-02-27T21:48:16.267Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650553677Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552}
109822024-02-27T21:48:16.274Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650596232Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552}
109832024-02-27T21:48:16.275Z{"msg":"Crucible 92628569-8198-41fd-a5e1-983bdd36c9cd has session id: a566c746-ca75-4fec-9ce1-2ea73686217c","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
109842024-02-27T21:48:16.275Z2024-02-27T21:48:14.650636058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109852024-02-27T21:48:16.275Z{"msg":"Upstairs opts: Upstairs UUID: 92628569-8198-41fd-a5e1-983bdd36c9cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.65070045Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109862024-02-27T21:48:16.275Z{"msg":"Crucible stats registered with UUID: 92628569-8198-41fd-a5e1-983bdd36c9cd","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650749244Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109872024-02-27T21:48:16.275Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650780271Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552}
109882024-02-27T21:48:16.275Z{"msg":"92628569-8198-41fd-a5e1-983bdd36c9cd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650832915Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109892024-02-27T21:48:16.275Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650863091Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109902024-02-27T21:48:16.276Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650915715Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109912024-02-27T21:48:16.276Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650946561Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109922024-02-27T21:48:16.276Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.650987586Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109932024-02-27T21:48:16.276Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.651012323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109942024-02-27T21:48:16.276Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.651036111Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109952024-02-27T21:48:16.276Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.651065467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109962024-02-27T21:48:16.276Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.651116221Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109972024-02-27T21:48:16.276Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.651167995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109982024-02-27T21:48:16.277Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.65121077Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
109992024-02-27T21:48:16.277Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.651250645Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110002024-02-27T21:48:16.277Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.651323577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110012024-02-27T21:48:16.277Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.652030513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110022024-02-27T21:48:16.277Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.652070479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110032024-02-27T21:48:16.277Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.652098046Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110042024-02-27T21:48:16.277Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.652191484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110052024-02-27T21:48:16.277Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.65222548Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110062024-02-27T21:48:16.278Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.652273135Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110072024-02-27T21:48:16.278Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.652660629Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110082024-02-27T21:48:16.278Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.652701494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110092024-02-27T21:48:16.278Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.652893622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110102024-02-27T21:48:16.278Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653135813Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110112024-02-27T21:48:16.278Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653186047Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110122024-02-27T21:48:16.279Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653215304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110132024-02-27T21:48:16.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653258029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110142024-02-27T21:48:16.279Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653287075Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110152024-02-27T21:48:16.279Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653323581Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110162024-02-27T21:48:16.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653359157Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110172024-02-27T21:48:16.279Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653400682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110182024-02-27T21:48:16.279Z{"msg":"[0]R flush_numbers[0..12]: [172, 191, 190, 190, 190, 191, 191, 189, 189, 190, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653431768Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110192024-02-27T21:48:16.280Z{"msg":"[0]R generation[0..12]: [90, 99, 98, 98, 98, 99, 99, 97, 97, 98, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653459055Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110202024-02-27T21:48:16.280Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653474533Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110212024-02-27T21:48:16.280Z{"msg":"[1]R flush_numbers[0..12]: [172, 191, 190, 190, 190, 191, 191, 189, 189, 190, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653495781Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110222024-02-27T21:48:16.280Z{"msg":"[1]R generation[0..12]: [90, 99, 98, 98, 98, 99, 99, 97, 97, 98, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653511149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110232024-02-27T21:48:16.280Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
110242024-02-27T21:48:16.280Z,"time":"2024-02-27T21:48:14.653545265Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110252024-02-27T21:48:16.281Z{"msg":"[2]R flush_numbers[0..12]: [172, 191, 190, 190, 190, 191, 191, 189, 189, 190, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653573812Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110262024-02-27T21:48:16.281Z{"msg":"[2]R generation[0..12]: [90, 99, 98, 98, 98, 99, 99, 97, 97, 98, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653605708Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110272024-02-27T21:48:16.281Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653639294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110282024-02-27T21:48:16.281Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.65367438Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110292024-02-27T21:48:16.281Z{"msg":"Generation requested: 102 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653705416Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110302024-02-27T21:48:16.281Z{"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653729163Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110312024-02-27T21:48:16.281Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.65375546Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"":"downstairs","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110322024-02-27T21:48:16.282Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653778977Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110332024-02-27T21:48:16.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653804794Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"0","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110342024-02-27T21:48:16.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653828572Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"1","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110352024-02-27T21:48:16.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653867997Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"client":"2","":"downstairs","session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110362024-02-27T21:48:16.282Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653893224Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110372024-02-27T21:48:16.282Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653919101Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110382024-02-27T21:48:16.282Z{"msg":"92628569-8198-41fd-a5e1-983bdd36c9cd is now active with session: a566c746-ca75-4fec-9ce1-2ea73686217c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653942458Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110392024-02-27T21:48:16.283Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653968635Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552,"session_id":"a566c746-ca75-4fec-9ce1-2ea73686217c"}
110402024-02-27T21:48:16.283Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:14.653991132Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552}
110412024-02-27T21:48:16.283ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110422024-02-27T21:48:16.283ZLoading write count information from file "/tmp/test_repair_verify.data"
110432024-02-27T21:48:16.283ZRun Repair workload
110442024-02-27T21:48:16.288Z01/30 Write block 382 len 16384 data: 7 7 8 8
110452024-02-27T21:48:16.288Z02/30 Read block 451 len 24576
110462024-02-27T21:48:16.288Z03/30 Write block 185 len 28672 data: 9 10 10 11 11 9 9
110472024-02-27T21:48:16.288Z04/30 Read block 465 len 4096
110482024-02-27T21:48:16.288Z05/30 Read block 225 len 36864
110492024-02-27T21:48:16.289Z06/30 Read block 160 len 16384
110502024-02-27T21:48:16.289Z07/30 Write block 582 len 20480 data: 8 8 8 7 8
110512024-02-27T21:48:16.289Z08/30 Flush
110522024-02-27T21:48:16.289Z09/30 Write block 251 len 40960 data: 6 8 8 7 10 10 7 9 8 9
110532024-02-27T21:48:16.289Z10/30 Read block 110 len 36864
110542024-02-27T21:48:16.289Z11/30 Read block 77 len 20480
110552024-02-27T21:48:16.289Z12/30 Read block 534 len 40960
110562024-02-27T21:48:16.289Z13/30 Write block 299 len 36864 data: 7 7 7 9 7 10 9 10 13
110572024-02-27T21:48:16.290Z14/30 Read block 369 len 20480
110582024-02-27T21:48:16.290Z15/30 Write block 232 len 32768 data: 13 12 12 12 12 7 7 7
110592024-02-27T21:48:16.294Z16/30 Write block 513 len 40960 data: 13 11 9 10 10 6 5 6 3 5
110602024-02-27T21:48:16.295Z17/30 Write block 517 len 4096 data: 11
110612024-02-27T21:48:16.295Z18/30 Read block 126 len 24576
110622024-02-27T21:48:16.295Z19/30 Write block 324 len 24576 data: 6 6 5 6 7 7
110632024-02-27T21:48:16.295Z20/30 Read block 115 len 20480
110642024-02-27T21:48:16.295Z21/30 Read block 409 len 8192
110652024-02-27T21:48:16.296Z22/30 Read block 263 len 40960
110662024-02-27T21:48:16.300Z23/30 Read block 551 len 24576
110672024-02-27T21:48:16.300Z24/30 Flush
110682024-02-27T21:48:16.300Z25/30 Write block 288 len 16384 data: 7 10 10 10
110692024-02-27T21:48:16.306Z26/30 Flush
110702024-02-27T21:48:16.306Z27/30 Write block 564 len 28672 data: 7 5 4 3 5 6 5
110712024-02-27T21:48:16.306Z28/30 Write block 8 len 20480 data: 9 12 13 13 12
110722024-02-27T21:48:16.307Z29/30 Write block 562 len 12288 data: 12 11 8
110732024-02-27T21:48:16.311Z30/30 Read block 592 len 24576
110742024-02-27T21:48:16.311Z----------------------------------------------------------------
110752024-02-27T21:48:16.311Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110762024-02-27T21:48:16.311ZStates: Active Active Active
110772024-02-27T21:48:16.311ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110782024-02-27T21:48:16.312Z 1 Acked 1000 Write 4 Done Done Done false
110792024-02-27T21:48:16.312Z 2 Acked 1001 Read 6 Done Done Done false
110802024-02-27T21:48:16.312Z 3 Acked 1002 Write 7 Done Sent Done false
110812024-02-27T21:48:16.312Z 4 Acked 1003 Read 1 Done Sent Done false
110822024-02-27T21:48:16.312Z 5 Acked 1004 Read 9 Done Sent Done false
110832024-02-27T21:48:16.312Z 6 Acked 1005 Read 4 Done Sent Done false
110842024-02-27T21:48:16.312Z 7 Acked 1006 Write 5 Done Sent Done false
110852024-02-27T21:48:16.312Z 8 Acked 1007 Flush 0 Done Sent Done false
110862024-02-27T21:48:16.312Z 9 Acked 1008 Write 10 Done Sent Done false
110872024-02-27T21:48:16.312Z 10 Acked 1009 Read 9 Done Sent Done false
110882024-02-27T21:48:16.312Z 11 Acked 1010 Read 5 Done Sent Done false
110892024-02-27T21:48:16.313Z 12 Acked 1011 Read 10 Done Sent Done false
110902024-02-27T21:48:16.313Z 13 Acked 1012 Write 9 Done Sent Done false
110912024-02-27T21:48:16.313Z 14 Acked 1013 Read 5 Done Sent Done false
110922024-02-27T21:48:16.313Z 15 Acked 1014 Write 8 Done Sent Done false
110932024-02-27T21:48:16.313Z 16 Acked 1015 Write 10 Done Sent Done false
110942024-02-27T21:48:16.313Z 17 Acked 1016 Write 1 Done Sent Done false
110952024-02-27T21:48:16.313Z 18 Acked 1017 Read 6 Done Sent Done false
110962024-02-27T21:48:16.313Z 19 Acked 1018 Write 6 Done Sent Done false
110972024-02-27T21:48:16.314Z 20 Acked 1019 Read 5 Done Sent Done false
110982024-02-27T21:48:16.314Z 21 Acked 1020 Read 2 Done Sent Done false
110992024-02-27T21:48:16.314Z 22 Acked 1021 Read 10 Done Sent Done false
111002024-02-27T21:48:16.314Z 23 Acked 1022 Read 6 Done Sent Done false
111012024-02-27T21:48:16.314Z 24 Acked 1023 Flush 0 Done Sent Done false
111022024-02-27T21:48:16.314Z 25 Acked 1024 Write 4 Done Sent Done false
111032024-02-27T21:48:16.314Z 26 Acked 1025 Flush 0 Done Sent Done false
111042024-02-27T21:48:16.314Z 27 Acked 1026 Write 7 Done Sent Done false
111052024-02-27T21:48:16.314Z 28 Acked 1027 Write 5 Done Sent Done false
111062024-02-27T21:48:16.314Z 29 Acked 1028 Write 3 Done Sent Done false
111072024-02-27T21:48:16.315Z 30 Acked 1029 Read 6 Done Sent Sent false
111082024-02-27T21:48:16.315Z STATES DS:0 DS:1 DS:2 TOTAL
111092024-02-27T21:48:16.315Z New 0 0 0 0
111102024-02-27T21:48:16.315Z Sent 0 28 1 29
111112024-02-27T21:48:16.315Z Done 30 2 29 61
111122024-02-27T21:48:16.315Z Skipped 0 0 0 0
111132024-02-27T21:48:16.315Z Error 0 0 0 0
111142024-02-27T21:48:16.315ZLast Flush: 1025 0 1025
111152024-02-27T21:48:16.316ZDownstairs last five completed:
111162024-02-27T21:48:16.316ZUpstairs last five completed: 30 29 28 27 26
111172024-02-27T21:48:16.316Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:14.68907361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1552}
111182024-02-27T21:48:16.316ZWrote out file "/tmp/test_repair_verify.data"
111192024-02-27T21:48:16.316Z
111202024-02-27T21:48:16.320ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111212024-02-27T21:48:16.328ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111222024-02-27T21:48:16.329Z 0 000-019 90 90 90 172 172 172 T F T <---
111232024-02-27T21:48:16.333Z 1 020-039 99 99 99 191 191 191 F F F
111242024-02-27T21:48:16.333Z 2 040-059 98 98 98 190 190 190 F F F
111252024-02-27T21:48:16.333Z 3 060-079 98 98 98 190 190 190 F F F
111262024-02-27T21:48:16.334Z 4 080-099 98 98 98 190 190 190 F F F
111272024-02-27T21:48:16.334Z 5 100-119 99 99 99 191 191 191 F F F
111282024-02-27T21:48:16.334Z 6 120-139 99 99 99 191 191 191 F F F
111292024-02-27T21:48:16.334Z 7 140-159 97 97 97 189 189 189 F F F
111302024-02-27T21:48:16.334Z 8 160-179 97 97 97 189 189 189 F F F
111312024-02-27T21:48:16.334Z 9 180-199 102 98 102 192 190 192 F F F <---
111322024-02-27T21:48:16.334Z 10 200-219 89 89 89 171 171 171 F F F
111332024-02-27T21:48:16.334Z 11 220-239 102 89 102 193 171 193 F F F <---
111342024-02-27T21:48:16.334Z 12 240-259 102 98 102 193 190 193 F F F <---
111352024-02-27T21:48:16.334Z 13 260-279 102 94 102 193 183 193 F F F <---
111362024-02-27T21:48:16.335Z 14 280-299 102 96 102 194 185 194 F F F <---
111372024-02-27T21:48:16.335Z 15 300-319 102 96 102 193 188 193 F F F <---
111382024-02-27T21:48:16.335Z 16 320-339 102 99 102 193 191 193 F F F <---
111392024-02-27T21:48:16.335Z 17 340-359 99 99 99 191 191 191 F F F
111402024-02-27T21:48:16.335Z 18 360-379 99 99 99 191 191 191 F F F
111412024-02-27T21:48:16.335Z 19 380-399 102 99 102 192 191 192 F T F <---
111422024-02-27T21:48:16.335Z 20 400-419 99 99 99 191 191 191 F F F
111432024-02-27T21:48:16.336Z 21 420-439 98 98 98 190 190 190 F F F
111442024-02-27T21:48:16.336Z 22 440-459 99 99 99 191 191 191 F F F
111452024-02-27T21:48:16.336Z 23 460-479 99 99 99 191 191 191 F F F
111462024-02-27T21:48:16.336Z 24 480-499 99 99 99 191 191 191 F F F
111472024-02-27T21:48:16.336Z 25 500-519 102 99 102 193 191 193 F F F <---
111482024-02-27T21:48:16.336Z 26 520-539 102 96 102 193 188 193 F F F <---
111492024-02-27T21:48:16.336Z 27 540-559 97 97 97 189 189 189 F F F
111502024-02-27T21:48:16.336Z 28 560-579 97 97 97 189 189 189 T F T <---
111512024-02-27T21:48:16.336Z 29 580-599 102 99 102 192 191 192 F F F <---
111522024-02-27T21:48:16.336ZMax gen: 102, Max flush: 194
111532024-02-27T21:48:16.337ZError: Difference in extent metadata found!
111542024-02-27T21:48:16.337ZA Difference in extent metadata is expected here
111552024-02-27T21:48:16.337ZOn loop 51
111562024-02-27T21:48:16.337Z
111572024-02-27T21:48:16.342ZVerifying data now
111582024-02-27T21:48:20.969ZNo change after verify
111592024-02-27T21:48:20.969ZLoop: 51 Downstairs dump after verify (and repair):
111602024-02-27T21:48:20.983ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111612024-02-27T21:48:20.984Z 0 000-019 103 103 103 195 195 195 F F F
111622024-02-27T21:48:20.984Z 1 020-039 99 99 99 191 191 191 F F F
111632024-02-27T21:48:20.984Z 2 040-059 98 98 98 190 190 190 F F F
111642024-02-27T21:48:20.984Z 3 060-079 98 98 98 190 190 190 F F F
111652024-02-27T21:48:20.984Z 4 080-099 98 98 98 190 190 190 F F F
111662024-02-27T21:48:20.985Z 5 100-119 99 99 99 191 191 191 F F F
111672024-02-27T21:48:20.985Z 6 120-139 99 99 99 191 191 191 F F F
111682024-02-27T21:48:20.985Z 7 140-159 97 97 97 189 189 189 F F F
111692024-02-27T21:48:20.985Z 8 160-179 97 97 97 189 189 189 F F F
111702024-02-27T21:48:20.989Z 9 180-199 102 102 102 192 192 192 F F F
111712024-02-27T21:48:20.989Z 10 200-219 89 89 89 171 171 171 F F F
111722024-02-27T21:48:20.989Z 11 220-239 102 102 102 193 193 193 F F F
111732024-02-27T21:48:20.990Z 12 240-259 102 102 102 193 193 193 F F F
111742024-02-27T21:48:20.990Z 13 260-279 102 102 102 193 193 193 F F F
111752024-02-27T21:48:20.990Z 14 280-299 102 102 102 194 194 194 F F F
111762024-02-27T21:48:20.990Z 15 300-319 102 102 102 193 193 193 F F F
111772024-02-27T21:48:20.990Z 16 320-339 102 102 102 193 193 193 F F F
111782024-02-27T21:48:20.990Z 17 340-359 99 99 99 191 191 191 F F F
111792024-02-27T21:48:20.990Z 18 360-379 99 99 99 191 191 191 F F F
111802024-02-27T21:48:20.990Z 19 380-399 102 102 102 192 192 192 F F F
111812024-02-27T21:48:20.991Z 20 400-419 99 99 99 191 191 191 F F F
111822024-02-27T21:48:20.991Z 21 420-439 98 98 98 190 190 190 F F F
111832024-02-27T21:48:20.991Z 22 440-459 99 99 99 191 191 191 F F F
111842024-02-27T21:48:20.991Z 23 460-479 99 99 99 191 191 191 F F F
111852024-02-27T21:48:20.991Z 24 480-499 99 99 99 191 191 191 F F F
111862024-02-27T21:48:20.991Z 25 500-519 102 102 102 193 193 193 F F F
111872024-02-27T21:48:20.992Z 26 520-539 102 102 102 193 193 193 F F F
111882024-02-27T21:48:20.992Z 27 540-559 97 97 97 189 189 189 F F F
111892024-02-27T21:48:20.992Z 28 560-579 103 103 103 195 195 195 F F F
111902024-02-27T21:48:20.992Z 29 580-599 102 102 102 192 192 192 F F F
111912024-02-27T21:48:20.992ZMax gen: 103, Max flush: 195
111922024-02-27T21:48:20.992Z
111932024-02-27T21:48:20.992ZBegin loop 52
111942024-02-27T21:48:20.992ZDownstairs to restart: 2
111952024-02-27T21:48:21.010Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.388586813Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560}
111962024-02-27T21:48:21.010Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
111972024-02-27T21:48:21.010Z,"time":"2024-02-27T21:48:19.388922064Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560}
111982024-02-27T21:48:21.019Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.388961519Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560}
111992024-02-27T21:48:21.020Z{"msg":"Crucible 11d0aaf8-d82f-48e8-abc4-afdf72d10d4d has session id: 08ad5cc8-d2b4-4afd-ac9c-d93010cd494d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389013393Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112002024-02-27T21:48:21.020Z{"msg":"Upstairs opts: Upstairs UUID: 11d0aaf8-d82f-48e8-abc4-afdf72d10d4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389060017Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112012024-02-27T21:48:21.020Z{"msg":"Crucible stats registered with UUID: 11d0aaf8-d82f-48e8-abc4-afdf72d10d4d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389081365Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112022024-02-27T21:48:21.020Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389100583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560}
112032024-02-27T21:48:21.020Z{"msg":"11d0aaf8-d82f-48e8-abc4-afdf72d10d4d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.3891214Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112042024-02-27T21:48:21.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389137428Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112052024-02-27T21:48:21.020Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389167905Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112062024-02-27T21:48:21.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389226748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112072024-02-27T21:48:21.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389269483Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112082024-02-27T21:48:21.021Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389316177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112092024-02-27T21:48:21.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389368481Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112102024-02-27T21:48:21.021Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389402667Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112112024-02-27T21:48:21.021Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389441743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112122024-02-27T21:48:21.021Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389499086Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112132024-02-27T21:48:21.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389592455Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112142024-02-27T21:48:21.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389671425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112152024-02-27T21:48:21.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.389728729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112162024-02-27T21:48:21.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.390327168Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112172024-02-27T21:48:21.022Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.390362204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112182024-02-27T21:48:21.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.390388451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112192024-02-27T21:48:21.022Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.390413118Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112202024-02-27T21:48:21.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.390511057Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112212024-02-27T21:48:21.022Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.39056505Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112222024-02-27T21:48:21.022Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.390895941Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112232024-02-27T21:48:21.022Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.39107437Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112242024-02-27T21:48:21.022Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391297324Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112252024-02-27T21:48:21.023Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391442847Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112262024-02-27T21:48:21.023Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391475943Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112272024-02-27T21:48:21.023Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.39150183Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112282024-02-27T21:48:21.023Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391533236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112292024-02-27T21:48:21.023Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391558843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112302024-02-27T21:48:21.023Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391578271Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112312024-02-27T21:48:21.023Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391790066Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112322024-02-27T21:48:21.023Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.39183736Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112332024-02-27T21:48:21.023Z{"msg":"[0]R flush_numbers[0..12]: [195, 191, 190, 190, 190, 191, 191, 189, 189, 192, 171, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391865207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112342024-02-27T21:48:21.023Z{"msg":"[0]R generation[0..12]: [103, 99, 98, 98, 98, 99, 99, 97, 97, 102, 89, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391888304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112352024-02-27T21:48:21.024Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391905872Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112362024-02-27T21:48:21.024Z{"msg":"[1]R flush_numbers[0..12]: [195, 191, 190, 190, 190, 191, 191, 189, 189, 192, 171, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.39192127Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112372024-02-27T21:48:21.024Z{"msg":"[1]R generation[0..12]: [103, 99, 98, 98, 98, 99, 99, 97, 97, 102, 89, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391935469Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112382024-02-27T21:48:21.024Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.391949827Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112392024-02-27T21:48:21.024Z{"msg":"[2]R flush_numbers[0..12]: [195, 191, 190, 190, 190, 191, 191, 189, 189, 192, 171, 193]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
112402024-02-27T21:48:21.024Ztime":"2024-02-27T21:48:19.391982743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112412024-02-27T21:48:21.024Z{"msg":"[2]R generation[0..12]: [103, 99, 98, 98, 98, 99, 99, 97, 97, 102, 89, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392014629Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112422024-02-27T21:48:21.024Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392041746Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112432024-02-27T21:48:21.024Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392056264Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112442024-02-27T21:48:21.024Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392069923Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112452024-02-27T21:48:21.025Z{"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.39209028Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112462024-02-27T21:48:21.025Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392103729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"":"downstairs","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112472024-02-27T21:48:21.025Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392117787Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112482024-02-27T21:48:21.025Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392137455Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"0","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112492024-02-27T21:48:21.025Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392151283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"1","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112502024-02-27T21:48:21.025Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392164572Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"client":"2","":"downstairs","session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112512024-02-27T21:48:21.025Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392184729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112522024-02-27T21:48:21.025Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392197438Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112532024-02-27T21:48:21.025Z{"msg":"11d0aaf8-d82f-48e8-abc4-afdf72d10d4d is now active with session: 08ad5cc8-d2b4-4afd-ac9c-d93010cd494d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392210566Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112542024-02-27T21:48:21.025Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392222965Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560,"session_id":"08ad5cc8-d2b4-4afd-ac9c-d93010cd494d"}
112552024-02-27T21:48:21.026Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:19.392235403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560}
112562024-02-27T21:48:21.026ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112572024-02-27T21:48:21.026ZLoading write count information from file "/tmp/test_repair_verify.data"
112582024-02-27T21:48:21.026ZRun Repair workload
112592024-02-27T21:48:21.026Z01/30 Write block 340 len 36864 data: 12 12 10 9 9 7 7 8 8
112602024-02-27T21:48:21.026Z02/30 Read block 288 len 12288
112612024-02-27T21:48:21.044Z03/30 Write block 13 len 40960 data: 11 10 11 11 10 8 7 7 7 6
112622024-02-27T21:48:21.044Z04/30 Write block 333 len 20480 data: 10 10 12 9 10
112632024-02-27T21:48:21.044Z05/30 Read block 507 len 16384
112642024-02-27T21:48:21.044Z06/30 Read block 208 len 8192
112652024-02-27T21:48:21.044Z07/30 Read block 319 len 4096
112662024-02-27T21:48:21.045Z08/30 Write block 455 len 4096 data: 5
112672024-02-27T21:48:21.045Z09/30 Read block 119 len 4096
112682024-02-27T21:48:21.045Z10/30 Read block 102 len 16384
112692024-02-27T21:48:21.045Z11/30 Write block 2 len 32768 data: 4 4 6 6 8 9 10 13
112702024-02-27T21:48:21.045Z12/30 Write block 78 len 40960 data: 6 6 5 5 4 5 6 6 6 7
112712024-02-27T21:48:21.045Z13/30 Read block 418 len 24576
112722024-02-27T21:48:21.045Z14/30 Read block 457 len 36864
112732024-02-27T21:48:21.045Z15/30 Write block 164 len 8192 data: 6 6
112742024-02-27T21:48:21.046Z16/30 Write block 518 len 28672 data: 7 6 7 4 6 4 4
112752024-02-27T21:48:21.046Z17/30 Read block 442 len 24576
112762024-02-27T21:48:21.046Z18/30 Write block 230 len 40960 data: 10 12 14 13 13 13 13 8 8 8
112772024-02-27T21:48:21.046Z19/30 Write block 568 len 20480 data: 6 7 6 7 7
112782024-02-27T21:48:21.046Z20/30 Write block 124 len 32768 data: 12 9 9 9 10 9 9 7
112792024-02-27T21:48:21.046Z21/30 Read block 237 len 8192
112802024-02-27T21:48:21.046Z22/30 Read block 84 len 36864
112812024-02-27T21:48:21.065Z23/30 Write block 272 len 24576 data: 9 7 7 6 8 7
112822024-02-27T21:48:21.065Z24/30 Write block 20 len 8192 data: 8 8
112832024-02-27T21:48:21.065Z25/30 Read block 254 len 40960
112842024-02-27T21:48:21.065Z26/30 Flush
112852024-02-27T21:48:21.065Z27/30 Write block 360 len 24576 data: 8 5 5 9 9 10
112862024-02-27T21:48:21.065Z28/30 Read block 580 len 28672
112872024-02-27T21:48:21.065Z29/30 Write block 513 len 4096 data: 14
112882024-02-27T21:48:21.066Z30/30 Read block 392 len 8192
112892024-02-27T21:48:21.066Z----------------------------------------------------------------
112902024-02-27T21:48:21.066Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
112912024-02-27T21:48:21.066ZStates: Active Active Active
112922024-02-27T21:48:21.066ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112932024-02-27T21:48:21.066Z 1 Acked 1000 Write 9 Done Done Done false
112942024-02-27T21:48:21.066Z 2 Acked 1001 Read 3 Done Done Done false
112952024-02-27T21:48:21.067Z 3 Acked 1002 Write 10 Done Done Sent false
112962024-02-27T21:48:21.067Z 4 Acked 1003 Write 5 Done Done Sent false
112972024-02-27T21:48:21.067Z 5 Acked 1004 Read 4 Done Done Sent false
112982024-02-27T21:48:21.067Z 6 Acked 1005 Read 2 Done Done Sent false
112992024-02-27T21:48:21.067Z 7 Acked 1006 Read 1 Done Done Sent false
113002024-02-27T21:48:21.068Z 8 Acked 1007 Write 1 Done Done Sent false
113012024-02-27T21:48:21.068Z 9 Acked 1008 Read 1 Done Done Sent false
113022024-02-27T21:48:21.068Z 10 Acked 1009 Read 4 Done Done Sent false
113032024-02-27T21:48:21.068Z 11 Acked 1010 Write 8 Done Done Sent false
113042024-02-27T21:48:21.068Z 12 Acked 1011 Write 10 Done Done Sent false
113052024-02-27T21:48:21.068Z 13 Acked 1012 Read 6 Done Done Sent false
113062024-02-27T21:48:21.068Z 14 Acked 1013 Read 9 Done Done Sent false
113072024-02-27T21:48:21.069Z 15 Acked 1014 Write 2 Done Done Sent false
113082024-02-27T21:48:21.069Z 16 Acked 1015 Write 7 Done Done Sent false
113092024-02-27T21:48:21.069Z 17 Acked 1016 Read 6 Done Done Sent false
113102024-02-27T21:48:21.069Z 18 Acked 1017 Write 10 Done Done Sent false
113112024-02-27T21:48:21.069Z 19 Acked 1018 Write 5 Done Done Sent false
113122024-02-27T21:48:21.069Z 20 Acked 1019 Write 8 Done Done Sent false
113132024-02-27T21:48:21.070Z 21 Acked 1020 Read 2 Done Done Sent false
113142024-02-27T21:48:21.070Z 22 Acked 1021 Read 9 Done Done Sent false
113152024-02-27T21:48:21.070Z 23 Acked 1022 Write 6 Done Done Sent false
113162024-02-27T21:48:21.070Z 24 Acked 1023 Write 2 Done Done Sent false
113172024-02-27T21:48:21.070Z 25 Acked 1024 Read 10 Done Done Sent false
113182024-02-27T21:48:21.070Z 26 Acked 1025 Flush 0 Done Done Sent false
113192024-02-27T21:48:21.070Z 27 Acked 1026 Write 6 Done Done Sent false
113202024-02-27T21:48:21.071Z 28 Acked 1027 Read 7 Done Done Sent false
113212024-02-27T21:48:21.071Z 29 Acked 1028 Write 1 Done Done Sent false
113222024-02-27T21:48:21.071Z 30 Acked 1029 Read 2 Done Sent Sent false
113232024-02-27T21:48:21.071Z STATES DS:0 DS:1 DS:2 TOTAL
113242024-02-27T21:48:21.071Z New 0 0 0 0
113252024-02-27T21:48:21.071Z Sent 0 1 28 29
113262024-02-27T21:48:21.071Z Done 30 29 2 61
113272024-02-27T21:48:21.071Z Skipped 0 0 0 0
113282024-02-27T21:48:21.071Z Error 0 0 0 0
113292024-02-27T21:48:21.071ZLast Flush: 1025 1025 0
113302024-02-27T21:48:21.071ZDownstairs last five completed:
113312024-02-27T21:48:21.071ZUpstairs last five completed: 30 29 28 27 26
113322024-02-27T21:48:21.072Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:19.425226667Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1560}
113332024-02-27T21:48:21.072ZWrote out file "/tmp/test_repair_verify.data"
113342024-02-27T21:48:21.072Z
113352024-02-27T21:48:21.072ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
113362024-02-27T21:48:21.077ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113372024-02-27T21:48:21.077Z 0 000-019 104 104 103 196 196 195 F F F <---
113382024-02-27T21:48:21.077Z 1 020-039 104 104 99 196 196 191 F F F <---
113392024-02-27T21:48:21.078Z 2 040-059 98 98 98 190 190 190 F F F
113402024-02-27T21:48:21.078Z 3 060-079 104 104 98 196 196 190 F F F <---
113412024-02-27T21:48:21.078Z 4 080-099 104 104 98 196 196 190 F F F <---
113422024-02-27T21:48:21.078Z 5 100-119 99 99 99 191 191 191 F F F
113432024-02-27T21:48:21.078Z 6 120-139 104 104 99 196 196 191 F F F <---
113442024-02-27T21:48:21.078Z 7 140-159 97 97 97 189 189 189 F F F
113452024-02-27T21:48:21.078Z 8 160-179 104 104 97 196 196 189 F F F <---
113462024-02-27T21:48:21.078Z 9 180-199 102 102 102 192 192 192 F F F
113472024-02-27T21:48:21.078Z 10 200-219 89 89 89 171 171 171 F F F
113482024-02-27T21:48:21.079Z 11 220-239 104 104 102 196 196 193 F F F <---
113492024-02-27T21:48:21.079Z 12 240-259 102 102 102 193 193 193 F F F
113502024-02-27T21:48:21.079Z 13 260-279 104 104 102 196 196 193 F F F <---
113512024-02-27T21:48:21.079Z 14 280-299 102 102 102 194 194 194 F F F
113522024-02-27T21:48:21.079Z 15 300-319 102 102 102 193 193 193 F F F
113532024-02-27T21:48:21.079Z 16 320-339 104 104 102 196 196 193 F F F <---
113542024-02-27T21:48:21.079Z 17 340-359 104 104 99 196 196 191 F F T <---
113552024-02-27T21:48:21.079Z 18 360-379 99 99 99 191 191 191 T T F <---
113562024-02-27T21:48:21.080Z 19 380-399 102 102 102 192 192 192 F F F
113572024-02-27T21:48:21.080Z 20 400-419 99 99 99 191 191 191 F F F
113582024-02-27T21:48:21.080Z 21 420-439 98 98 98 190 190 190 F F F
113592024-02-27T21:48:21.080Z 22 440-459 104 104 99 196 196 191 F F F <---
113602024-02-27T21:48:21.080Z 23 460-479 99 99 99 191 191 191 F F F
113612024-02-27T21:48:21.080Z 24 480-499 99 99 99 191 191 191 F F F
113622024-02-27T21:48:21.080Z 25 500-519 104 104 102 196 196 193 T T F <---
113632024-02-27T21:48:21.080Z 26 520-539 104 104 102 196 196 193 F F F <---
113642024-02-27T21:48:21.081Z 27 540-559 97 97 97 189 189 189 F F F
113652024-02-27T21:48:21.081Z 28 560-579 104 104 103 196 196 195 F F F <---
113662024-02-27T21:48:21.081Z 29 580-599 102 102 102 192 192 192 F F F
113672024-02-27T21:48:21.081ZMax gen: 104, Max flush: 196
113682024-02-27T21:48:21.081ZError: Difference in extent metadata found!
113692024-02-27T21:48:21.081ZA Difference in extent metadata is expected here
113702024-02-27T21:48:21.081ZOn loop 52
113712024-02-27T21:48:21.082Z
113722024-02-27T21:48:21.082ZVerifying data now
113732024-02-27T21:48:25.777ZNo change after verify
113742024-02-27T21:48:25.777ZLoop: 52 Downstairs dump after verify (and repair):
113752024-02-27T21:48:25.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113762024-02-27T21:48:25.792Z 0 000-019 104 104 104 196 196 196 F F F
113772024-02-27T21:48:25.792Z 1 020-039 104 104 104 196 196 196 F F F
113782024-02-27T21:48:25.792Z 2 040-059 98 98 98 190 190 190 F F F
113792024-02-27T21:48:25.793Z 3 060-079 104 104 104 196 196 196 F F F
113802024-02-27T21:48:25.793Z 4 080-099 104 104 104 196 196 196 F F F
113812024-02-27T21:48:25.793Z 5 100-119 99 99 99 191 191 191 F F F
113822024-02-27T21:48:25.793Z 6 120-139 104 104 104 196 196 196 F F F
113832024-02-27T21:48:25.793Z 7 140-159 97 97 97 189 189 189 F F F
113842024-02-27T21:48:25.793Z 8 160-179 104 104 104 196 196 196 F F F
113852024-02-27T21:48:25.793Z 9 180-199 102 102 102 192 192 192 F F F
113862024-02-27T21:48:25.793Z 10 200-219 89 89 89 171 171 171 F F F
113872024-02-27T21:48:25.793Z 11 220-239 104 104 104 196 196 196 F F F
113882024-02-27T21:48:25.799Z 12 240-259 102 102 102 193 193 193 F F F
113892024-02-27T21:48:25.799Z 13 260-279 104 104 104 196 196 196 F F F
113902024-02-27T21:48:25.799Z 14 280-299 102 102 102 194 194 194 F F F
113912024-02-27T21:48:25.799Z 15 300-319 102 102 102 193 193 193 F F F
113922024-02-27T21:48:25.800Z 16 320-339 104 104 104 196 196 196 F F F
113932024-02-27T21:48:25.800Z 17 340-359 104 104 104 196 196 196 F F F
113942024-02-27T21:48:25.800Z 18 360-379 105 105 105 197 197 197 F F F
113952024-02-27T21:48:25.800Z 19 380-399 102 102 102 192 192 192 F F F
113962024-02-27T21:48:25.800Z 20 400-419 99 99 99 191 191 191 F F F
113972024-02-27T21:48:25.800Z 21 420-439 98 98 98 190 190 190 F F F
113982024-02-27T21:48:25.800Z 22 440-459 104 104 104 196 196 196 F F F
113992024-02-27T21:48:25.800Z 23 460-479 99 99 99 191 191 191 F F F
114002024-02-27T21:48:25.800Z 24 480-499 99 99 99 191 191 191 F F F
114012024-02-27T21:48:25.800Z 25 500-519 105 105 105 197 197 197 F F F
114022024-02-27T21:48:25.800Z 26 520-539 104 104 104 196 196 196 F F F
114032024-02-27T21:48:25.800Z 27 540-559 97 97 97 189 189 189 F F F
114042024-02-27T21:48:25.800Z 28 560-579 104 104 104 196 196 196 F F F
114052024-02-27T21:48:25.801Z 29 580-599 102 102 102 192 192 192 F F F
114062024-02-27T21:48:25.801ZMax gen: 105, Max flush: 197
114072024-02-27T21:48:25.801Z
114082024-02-27T21:48:25.801ZBegin loop 53
114092024-02-27T21:48:25.801ZDownstairs to restart: 0
114102024-02-27T21:48:25.816Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.198796857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568}
114112024-02-27T21:48:25.816Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199085883Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568}
114122024-02-27T21:48:25.823Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199124328Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568}
114132024-02-27T21:48:25.823Z{"msg":"Crucible ca3ab95b-fff6-4165-9193-3ba9e9ffb517 has session id: 6757e6ff-fe3c-4a80-9bed-49a0734a0372","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199156964Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114142024-02-27T21:48:25.823Z{"msg":"Upstairs opts: Upstairs UUID: ca3ab95b-fff6-4165-9193-3ba9e9ffb517, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199189871Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114152024-02-27T21:48:25.823Z{"msg":"Crucible stats registered with UUID: ca3ab95b-fff6-4165-9193-3ba9e9ffb517","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199209538Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114162024-02-27T21:48:25.823ZCrucible runtime is spawned
114172024-02-27T21:48:25.823Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199337623Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568}
114182024-02-27T21:48:25.823Z{"msg":"ca3ab95b-fff6-4165-9193-3ba9e9ffb517 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199420153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114192024-02-27T21:48:25.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.19944704Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114202024-02-27T21:48:25.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199487356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114212024-02-27T21:48:25.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199522751Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114222024-02-27T21:48:25.824Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199546869Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114232024-02-27T21:48:25.824Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199592183Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114242024-02-27T21:48:25.824Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.19961916Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114252024-02-27T21:48:25.824Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199652696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114262024-02-27T21:48:25.824Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199679843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114272024-02-27T21:48:25.824Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199714199Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114282024-02-27T21:48:25.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.199856462Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114292024-02-27T21:48:25.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.20012681Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114302024-02-27T21:48:25.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.200163116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114312024-02-27T21:48:25.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.200530313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114322024-02-27T21:48:25.824Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.200576207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114332024-02-27T21:48:25.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.200710041Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114342024-02-27T21:48:25.824Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.200761305Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114352024-02-27T21:48:25.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.200902129Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114362024-02-27T21:48:25.824Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.200939774Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114372024-02-27T21:48:25.824Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.201114394Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114382024-02-27T21:48:25.824Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.20123185Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114392024-02-27T21:48:25.824Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.201604266Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114402024-02-27T21:48:25.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.201774066Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114412024-02-27T21:48:25.824Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.201814371Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114422024-02-27T21:48:25.824Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.201842218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114432024-02-27T21:48:25.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.201872775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114442024-02-27T21:48:25.824Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.20191377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114452024-02-27T21:48:25.824Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.201942346Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114462024-02-27T21:48:25.825Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.20208228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114472024-02-27T21:48:25.825Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202119975Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114482024-02-27T21:48:25.825Z{"msg":"[0]R flush_numbers[0..12]: [196, 196, 190, 196, 196, 191, 196, 189, 196, 192, 171, 196]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202136893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114492024-02-27T21:48:25.825Z{"msg":"[0]R generation[0..12]: [104, 104, 98, 104, 104, 99, 104, 97, 104, 102, 89, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202153091Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114502024-02-27T21:48:25.825Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.2021676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114512024-02-27T21:48:25.825Z{"msg":"[1]R flush_numbers[0..12]: [196, 196, 190, 196, 196, 191, 196, 189, 196, 192, 171, 196]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202195617Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114522024-02-27T21:48:25.825Z{"msg":"[1]R generation[0..12]: [104, 104, 98, 104, 104, 99, 104, 97, 104, 102, 89, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202223173Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114532024-02-27T21:48:25.825Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202241951Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114542024-02-27T21:48:25.825ZWait for a query_work_queue command to finish before sending IO
114552024-02-27T21:48:25.825Z{"msg":"[2]R flush_numbers[0..12]: [196, 196, 190, 196, 196, 191, 196, 189, 196, 192, 171, 196]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202268098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114562024-02-27T21:48:25.825Z{"msg":"[2]R generation[0..12]: [104, 104, 98, 104, 104, 99, 104, 97, 104, 102, 89, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202285316Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114572024-02-27T21:48:25.825Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202320702Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114582024-02-27T21:48:25.825Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202338709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114592024-02-27T21:48:25.825Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202361067Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114602024-02-27T21:48:25.825Z{"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202395593Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114612024-02-27T21:48:25.825Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202430239Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"":"downstairs","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114622024-02-27T21:48:25.825Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202466374Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114632024-02-27T21:48:25.825Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202499441Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114642024-02-27T21:48:25.825Z,"client":"0","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114652024-02-27T21:48:25.825Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202572562Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"1","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114662024-02-27T21:48:25.825Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202611907Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"client":"2","":"downstairs","session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114672024-02-27T21:48:25.830Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202638304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114682024-02-27T21:48:25.830Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202668141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114692024-02-27T21:48:25.830Z{"msg":"ca3ab95b-fff6-4165-9193-3ba9e9ffb517 is now active with session: 6757e6ff-fe3c-4a80-9bed-49a0734a0372","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202692148Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114702024-02-27T21:48:25.830Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202717365Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568,"session_id":"6757e6ff-fe3c-4a80-9bed-49a0734a0372"}
114712024-02-27T21:48:25.830Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:24.202740312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568}
114722024-02-27T21:48:25.830ZLoading write count information from file "/tmp/test_repair_verify.data"
114732024-02-27T21:48:25.830ZRun Repair workload
114742024-02-27T21:48:25.831Z01/30 Write block 557 len 4096 data: 16
114752024-02-27T21:48:25.831Z02/30 Write block 583 len 24576 data: 9 9 8 9 8 7
114762024-02-27T21:48:25.831Z03/30 Read block 225 len 24576
114772024-02-27T21:48:25.831Z04/30 Write block 469 len 4096 data: 9
114782024-02-27T21:48:25.831Z05/30 Write block 54 len 28672 data: 7 5 12 12 10 7 5
114792024-02-27T21:48:25.831Z06/30 Write block 59 len 12288 data: 8 6 4
114802024-02-27T21:48:25.831Z07/30 Flush
114812024-02-27T21:48:25.832Z08/30 Write block 583 len 12288 data: 10 10 9
114822024-02-27T21:48:25.832Z09/30 Write block 486 len 8192 data: 14 14
114832024-02-27T21:48:25.836Z10/30 Write block 375 len 12288 data: 3 3 4
114842024-02-27T21:48:25.836Z11/30 Read block 542 len 28672
114852024-02-27T21:48:25.836Z12/30 Flush
114862024-02-27T21:48:25.836Z13/30 Write block 328 len 8192 data: 8 8
114872024-02-27T21:48:25.836Z14/30 Read block 461 len 24576
114882024-02-27T21:48:25.836Z15/30 Read block 122 len 36864
114892024-02-27T21:48:25.839Z16/30 Write block 581 len 24576 data: 6 9 11 11 10 10
114902024-02-27T21:48:25.839Z17/30 Flush
114912024-02-27T21:48:25.839Z18/30 Write block 437 len 12288 data: 8 7 7
114922024-02-27T21:48:25.844Z19/30 Write block 213 len 32768 data: 9 9 8 9 12 12 10 9
114932024-02-27T21:48:25.844Z20/30 Read block 28 len 40960
114942024-02-27T21:48:25.844Z21/30 Read block 50 len 4096
114952024-02-27T21:48:25.844Z22/30 Read block 5 len 8192
114962024-02-27T21:48:25.849Z23/30 Read block 520 len 4096
114972024-02-27T21:48:25.849Z24/30 Flush
114982024-02-27T21:48:25.849Z25/30 Write block 230 len 28672 data: 11 13 15 14 14 14 14
114992024-02-27T21:48:25.849Z26/30 Read block 221 len 12288
115002024-02-27T21:48:25.849Z27/30 Read block 117 len 40960
115012024-02-27T21:48:25.855Z28/30 Write block 82 len 32768 data: 5 6 7 7 7 8 9 10
115022024-02-27T21:48:25.855Z29/30 Write block 550 len 4096 data: 11
115032024-02-27T21:48:25.855Z30/30 Write block 369 len 36864 data: 6 7 7 3 4 3 4 4 5
115042024-02-27T21:48:25.855Z----------------------------------------------------------------
115052024-02-27T21:48:25.855Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
115062024-02-27T21:48:25.856ZStates: Active Active Active
115072024-02-27T21:48:25.856ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
115082024-02-27T21:48:25.857Z 1 Acked 1000 Write 1 Done Done Done false
115092024-02-27T21:48:25.857Z 2 Acked 1001 Write 6 Sent Done Done false
115102024-02-27T21:48:25.857Z 3 Acked 1002 Read 6 Sent Done Done false
115112024-02-27T21:48:25.857Z 4 Acked 1003 Write 1 Sent Done Done false
115122024-02-27T21:48:25.858Z 5 Acked 1004 Write 7 Sent Done Done false
115132024-02-27T21:48:25.858Z 6 Acked 1005 Write 3 Sent Done Done false
115142024-02-27T21:48:25.858Z 7 Acked 1006 Flush 0 Sent Done Done false
115152024-02-27T21:48:25.858Z 8 Acked 1007 Write 3 Sent Done Done false
115162024-02-27T21:48:25.858Z 9 Acked 1008 Write 2 Sent Done Done false
115172024-02-27T21:48:25.858Z 10 Acked 1009 Write 3 Sent Done Done false
115182024-02-27T21:48:25.859Z 11 Acked 1010 Read 7 Sent Done Done false
115192024-02-27T21:48:25.859Z 12 Acked 1011 Flush 0 Sent Done Done false
115202024-02-27T21:48:25.859Z 13 Acked 1012 Write 2 Sent Done Done false
115212024-02-27T21:48:25.859Z 14 Acked 1013 Read 6 Sent Done Done false
115222024-02-27T21:48:25.859Z 15 Acked 1014 Read 9 Sent Done Done false
115232024-02-27T21:48:25.859Z 16 Acked 1015 Write 6 Sent Done Done false
115242024-02-27T21:48:25.860Z 17 Acked 1016 Flush 0 Sent Done Done false
115252024-02-27T21:48:25.860Z 18 Acked 1017 Write 3 Sent Done Done false
115262024-02-27T21:48:25.860Z 19 Acked 1018 Write 8 Sent Done Done false
115272024-02-27T21:48:25.860Z 20 Acked 1019 Read 10 Sent Done Done false
115282024-02-27T21:48:25.860Z 21 Acked 1020 Read 1 Sent Done Done false
115292024-02-27T21:48:25.861Z 22 Acked 1021 Read 2 Sent Done Done false
115302024-02-27T21:48:25.861Z 23 Acked 1022 Read 1 Sent Done Done false
115312024-02-27T21:48:25.861Z 24 Acked 1023 Flush 0 Sent Done Done false
115322024-02-27T21:48:25.862Z 25 Acked 1024 Write 7 Sent Done Done false
115332024-02-27T21:48:25.862Z 26 Acked 1025 Read 3 Sent Done Done false
115342024-02-27T21:48:25.862Z 27 Acked 1026 Read 10 Sent Done Done false
115352024-02-27T21:48:25.863Z 28 Acked 1027 Write 8 Sent Done Done false
115362024-02-27T21:48:25.863Z 29 Acked 1028 Write 1 Sent Done Done false
115372024-02-27T21:48:25.863Z 30 Acked 1029 Write 9 Sent Sent Sent false
115382024-02-27T21:48:25.863Z STATES DS:0 DS:1 DS:2 TOTAL
115392024-02-27T21:48:25.863Z New 0 0 0 0
115402024-02-27T21:48:25.864Z Sent 29 1 1 31
115412024-02-27T21:48:25.864Z Done 1 29 29 59
115422024-02-27T21:48:25.864Z Skipped 0 0 0 0
115432024-02-27T21:48:25.864Z Error 0 0 0 0
115442024-02-27T21:48:25.864ZLast Flush: 0 1023 1023
115452024-02-27T21:48:25.864ZDownstairs last five completed:
115462024-02-27T21:48:25.865ZUpstairs last five completed: 30 29 28 27 26
115472024-02-27T21:48:25.865Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:24.233780856Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1568}
115482024-02-27T21:48:25.865ZWrote out file "/tmp/test_repair_verify.data"
115492024-02-27T21:48:25.882Z
115502024-02-27T21:48:25.883ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115512024-02-27T21:48:25.898ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115522024-02-27T21:48:25.900Z 0 000-019 104 104 104 196 196 196 F F F
115532024-02-27T21:48:25.900Z 1 020-039 104 104 104 196 196 196 F F F
115542024-02-27T21:48:25.901Z 2 040-059 98 106 106 190 198 198 F F F <---
115552024-02-27T21:48:25.901Z 3 060-079 104 106 106 196 198 198 F F F <---
115562024-02-27T21:48:25.901Z 4 080-099 104 104 104 196 196 196 F T T <---
115572024-02-27T21:48:25.902Z 5 100-119 99 99 99 191 191 191 F F F
115582024-02-27T21:48:25.902Z 6 120-139 104 104 104 196 196 196 F F F
115592024-02-27T21:48:25.902Z 7 140-159 97 97 97 189 189 189 F F F
115602024-02-27T21:48:25.903Z 8 160-179 104 104 104 196 196 196 F F F
115612024-02-27T21:48:25.903Z 9 180-199 102 102 102 192 192 192 F F F
115622024-02-27T21:48:25.903Z 10 200-219 89 106 106 171 201 201 F F F <---
115632024-02-27T21:48:25.903Z 11 220-239 104 106 106 196 201 201 F T T <---
115642024-02-27T21:48:25.903Z 12 240-259 102 102 102 193 193 193 F F F
115652024-02-27T21:48:25.904Z 13 260-279 104 104 104 196 196 196 F F F
115662024-02-27T21:48:25.904Z 14 280-299 102 102 102 194 194 194 F F F
115672024-02-27T21:48:25.904Z 15 300-319 102 102 102 193 193 193 F F F
115682024-02-27T21:48:25.904Z 16 320-339 104 106 106 196 200 200 F F F <---
115692024-02-27T21:48:25.904Z 17 340-359 104 104 104 196 196 196 F F F
115702024-02-27T21:48:25.905Z 18 360-379 105 106 106 197 199 199 F T T <---
115712024-02-27T21:48:25.905Z 19 380-399 102 102 102 192 192 192 F F F
115722024-02-27T21:48:25.905Z 20 400-419 99 99 99 191 191 191 F F F
115732024-02-27T21:48:25.905Z 21 420-439 98 106 106 190 201 201 F F F <---
115742024-02-27T21:48:25.906Z 22 440-459 104 104 104 196 196 196 F F F
115752024-02-27T21:48:25.906Z 23 460-479 99 106 106 191 198 198 F F F <---
115762024-02-27T21:48:25.906Z 24 480-499 99 106 106 191 199 199 F F F <---
115772024-02-27T21:48:25.906Z 25 500-519 105 105 105 197 197 197 F F F
115782024-02-27T21:48:25.907Z 26 520-539 104 104 104 196 196 196 F F F
115792024-02-27T21:48:25.907Z 27 540-559 97 106 106 189 198 198 T T T <---
115802024-02-27T21:48:25.907Z 28 560-579 104 104 104 196 196 196 F F F
115812024-02-27T21:48:25.907Z 29 580-599 102 106 106 192 200 200 F F F <---
115822024-02-27T21:48:25.907ZMax gen: 106, Max flush: 201
115832024-02-27T21:48:25.908ZError: Difference in extent metadata found!
115842024-02-27T21:48:25.908ZA Difference in extent metadata is expected here
115852024-02-27T21:48:25.908ZOn loop 53
115862024-02-27T21:48:25.908Z
115872024-02-27T21:48:25.908ZVerifying data now
115882024-02-27T21:48:30.486ZNo change after verify
115892024-02-27T21:48:30.486ZLoop: 53 Downstairs dump after verify (and repair):
115902024-02-27T21:48:30.501ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115912024-02-27T21:48:30.501Z 0 000-019 104 104 104 196 196 196 F F F
115922024-02-27T21:48:30.501Z 1 020-039 104 104 104 196 196 196 F F F
115932024-02-27T21:48:30.501Z 2 040-059 106 106 106 198 198 198 F F F
115942024-02-27T21:48:30.501Z 3 060-079 106 106 106 198 198 198 F F F
115952024-02-27T21:48:30.502Z 4 080-099 107 107 107 202 202 202 F F F
115962024-02-27T21:48:30.502Z 5 100-119 99 99 99 191 191 191 F F F
115972024-02-27T21:48:30.502Z 6 120-139 104 104 104 196 196 196 F F F
115982024-02-27T21:48:30.502Z 7 140-159 97 97 97 189 189 189 F F F
115992024-02-27T21:48:30.502Z 8 160-179 104 104 104 196 196 196 F F F
116002024-02-27T21:48:30.502Z 9 180-199 102 102 102 192 192 192 F F F
116012024-02-27T21:48:30.502Z 10 200-219 106 106 106 201 201 201 F F F
116022024-02-27T21:48:30.507Z 11 220-239 107 107 107 202 202 202 F F F
116032024-02-27T21:48:30.507Z 12 240-259 102 102 102 193 193 193 F F F
116042024-02-27T21:48:30.508Z 13 260-279 104 104 104 196 196 196 F F F
116052024-02-27T21:48:30.508Z 14 280-299 102 102 102 194 194 194 F F F
116062024-02-27T21:48:30.508Z 15 300-319 102 102 102 193 193 193 F F F
116072024-02-27T21:48:30.508Z 16 320-339 106 106 106 200 200 200 F F F
116082024-02-27T21:48:30.508Z 17 340-359 104 104 104 196 196 196 F F F
116092024-02-27T21:48:30.508Z 18 360-379 107 107 107 202 202 202 F F F
116102024-02-27T21:48:30.508Z 19 380-399 102 102 102 192 192 192 F F F
116112024-02-27T21:48:30.509Z 20 400-419 99 99 99 191 191 191 F F F
116122024-02-27T21:48:30.509Z 21 420-439 106 106 106 201 201 201 F F F
116132024-02-27T21:48:30.509Z 22 440-459 104 104 104 196 196 196 F F F
116142024-02-27T21:48:30.509Z 23 460-479 106 106 106 198 198 198 F F F
116152024-02-27T21:48:30.509Z 24 480-499 106 106 106 199 199 199 F F F
116162024-02-27T21:48:30.509Z 25 500-519 105 105 105 197 197 197 F F F
116172024-02-27T21:48:30.509Z 26 520-539 104 104 104 196 196 196 F F F
116182024-02-27T21:48:30.510Z 27 540-559 107 107 107 202 202 202 F F F
116192024-02-27T21:48:30.510Z 28 560-579 104 104 104 196 196 196 F F F
116202024-02-27T21:48:30.510Z 29 580-599 106 106 106 200 200 200 F F F
116212024-02-27T21:48:30.510ZMax gen: 107, Max flush: 202
116222024-02-27T21:48:30.510Z
116232024-02-27T21:48:30.510ZBegin loop 54
116242024-02-27T21:48:30.510ZDownstairs to restart: 0
116252024-02-27T21:48:30.526Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907107963Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576}
116262024-02-27T21:48:30.526Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
116272024-02-27T21:48:30.527Z,"time":"2024-02-27T21:48:28.90747921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576}
116282024-02-27T21:48:30.534Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907537713Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576}
116292024-02-27T21:48:30.534Z{"msg":"Crucible fedaa690-13f0-4c40-9d17-8ac800b2f264 has session id: 48bc30f5-63f6-4590-84cb-27b8bb65a606","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907583937Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116302024-02-27T21:48:30.534Z{"msg":"Upstairs opts: Upstairs UUID: fedaa690-13f0-4c40-9d17-8ac800b2f264, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907637021Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116312024-02-27T21:48:30.535Z{"msg":"Crucible stats registered with UUID: fedaa690-13f0-4c40-9d17-8ac800b2f264","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907672547Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116322024-02-27T21:48:30.535Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907712182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576}
116332024-02-27T21:48:30.535Z{"msg":"fedaa690-13f0-4c40-9d17-8ac800b2f264 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.9077329Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116342024-02-27T21:48:30.535Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907758937Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116352024-02-27T21:48:30.535Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907789513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116362024-02-27T21:48:30.535Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907824019Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"1","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116372024-02-27T21:48:30.535Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907844957Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"1","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116382024-02-27T21:48:30.535Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907890131Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116392024-02-27T21:48:30.535Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907916468Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116402024-02-27T21:48:30.536Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.907964362Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116412024-02-27T21:48:30.536Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.908000798Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116422024-02-27T21:48:30.536Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.908030725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116432024-02-27T21:48:30.536Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.908088878Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116442024-02-27T21:48:30.536Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.908129503Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116452024-02-27T21:48:30.536Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.908209154Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116462024-02-27T21:48:30.536Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.908926149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"1","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116472024-02-27T21:48:30.537Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.908985372Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"1","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116482024-02-27T21:48:30.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.909015569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116492024-02-27T21:48:30.537Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.909056194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116502024-02-27T21:48:30.537Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.909493553Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"1","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116512024-02-27T21:48:30.537Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.909545866Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116522024-02-27T21:48:30.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.90968091Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116532024-02-27T21:48:30.537Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.909710127Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116542024-02-27T21:48:30.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910038558Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"1","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116552024-02-27T21:48:30.538Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910079673Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"1","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116562024-02-27T21:48:30.538Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910095491Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116572024-02-27T21:48:30.538Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910129727Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116582024-02-27T21:48:30.538Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910152425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116592024-02-27T21:48:30.538Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910167103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116602024-02-27T21:48:30.538Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910380728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116612024-02-27T21:48:30.538Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910824256Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116622024-02-27T21:48:30.538Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.91087112Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116632024-02-27T21:48:30.538Z{"msg":"[0]R flush_numbers[0..12]: [196, 196, 198, 198, 202, 191, 196, 189, 196, 192, 201, 202]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910891368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116642024-02-27T21:48:30.538Z{"msg":"[0]R generation[0..12]: [104, 104, 106, 106, 107, 99, 104, 97, 104, 102, 106, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910925294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116652024-02-27T21:48:30.539Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.9109591Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116662024-02-27T21:48:30.539Z{"msg":"[1]R flush_numbers[0..12]: [196, 196, 198, 198, 202, 191, 196, 189, 196, 192, 201, 202]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.910994945Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576Wait for a query_work_queue command to finish before sending IO
116672024-02-27T21:48:30.539Z,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116682024-02-27T21:48:30.539Z{"msg":"[1]R generation[0..12]: [104, 104, 106, 106, 107, 99, 104, 97, 104, 102, 106, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911025962Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116692024-02-27T21:48:30.539Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911054929Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116702024-02-27T21:48:30.539Z{"msg":"[2]R flush_numbers[0..12]: [196, 196, 198, 198, 202, 191, 196, 189, 196, 192, 201, 202]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911075166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116712024-02-27T21:48:30.539Z{"msg":"[2]R generation[0..12]: [104, 104, 106, 106, 107, 99, 104, 97, 104, 102, 106, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911099573Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116722024-02-27T21:48:30.540Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911138479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116732024-02-27T21:48:30.540Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911172695Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116742024-02-27T21:48:30.540Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911191422Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116752024-02-27T21:48:30.540Z{"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.91120915Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116762024-02-27T21:48:30.540Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911242466Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"":"downstairs","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116772024-02-27T21:48:30.540Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911261084Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116782024-02-27T21:48:30.540Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911291001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"0","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116792024-02-27T21:48:30.540Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911310368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"1","":"downstairsRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116802024-02-27T21:48:30.541Z","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116812024-02-27T21:48:30.541Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911350354Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"client":"2","":"downstairs","session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116822024-02-27T21:48:30.541Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911372651Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116832024-02-27T21:48:30.541Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911400118Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116842024-02-27T21:48:30.541Z{"msg":"fedaa690-13f0-4c40-9d17-8ac800b2f264 is now active with session: 48bc30f5-63f6-4590-84cb-27b8bb65a606","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911418476Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116852024-02-27T21:48:30.541Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911445723Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576,"session_id":"48bc30f5-63f6-4590-84cb-27b8bb65a606"}
116862024-02-27T21:48:30.541Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:28.911462721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576}
116872024-02-27T21:48:30.541ZLoading write count information from file "/tmp/test_repair_verify.data"
116882024-02-27T21:48:30.542ZRun Repair workload
116892024-02-27T21:48:30.542Z01/30 Write block 262 len 20480 data: 6 5 6 6 5
116902024-02-27T21:48:30.547Z02/30 Write block 215 len 28672 data: 9 10 13 13 11 10 9
116912024-02-27T21:48:30.547Z03/30 Read block 39 len 24576
116922024-02-27T21:48:30.547Z04/30 Read block 532 len 24576
116932024-02-27T21:48:30.548Z05/30 Write block 487 len 16384 data: 15 13 15 13
116942024-02-27T21:48:30.548Z06/30 Write block 69 len 36864 data: 7 6 6 8 5 6 7 6 7
116952024-02-27T21:48:30.548Z07/30 Read block 270 len 24576
116962024-02-27T21:48:30.548Z08/30 Write block 148 len 20480 data: 7 7 8 7 6
116972024-02-27T21:48:30.548Z09/30 Write block 10 len 40960 data: 14 14 13 12 11 12 12 11 9 8
116982024-02-27T21:48:30.548Z10/30 Read block 433 len 32768
116992024-02-27T21:48:30.548Z11/30 Write block 348 len 12288 data: 9 10 10
117002024-02-27T21:48:30.549Z12/30 Read block 266 len 20480
117012024-02-27T21:48:30.549Z13/30 Write block 410 len 40960 data: 5 4 6 6 6 7 8 11 10 9
117022024-02-27T21:48:30.549Z14/30 Write block 335 len 28672 data: 13 10 11 12 11 13 13
117032024-02-27T21:48:30.549Z15/30 Read block 508 len 20480
117042024-02-27T21:48:30.549Z16/30 Read block 540 len 40960
117052024-02-27T21:48:30.554Z17/30 Write block 176 len 32768 data: 11 10 12 10 13 15 15 12
117062024-02-27T21:48:30.554Z18/30 Read block 495 len 40960
117072024-02-27T21:48:30.554Z19/30 Read block 574 len 12288
117082024-02-27T21:48:30.554Z20/30 Read block 30 len 12288
117092024-02-27T21:48:30.554Z21/30 Write block 199 len 12288 data: 5 7 6
117102024-02-27T21:48:30.554Z22/30 Read block 139 len 24576
117112024-02-27T21:48:30.554Z23/30 Read block 383 len 40960
117122024-02-27T21:48:30.554Z24/30 Write block 121 len 8192 data: 10 10
117132024-02-27T21:48:30.559Z25/30 Write block 99 len 32768 data: 8 9 9 9 9 7 9 10
117142024-02-27T21:48:30.559Z26/30 Read block 86 len 12288
117152024-02-27T21:48:30.559Z27/30 Read block 417 len 40960
117162024-02-27T21:48:30.559Z28/30 Write block 255 len 24576 data: 11 11 8 10 9 10
117172024-02-27T21:48:30.559Z29/30 Write block 524 len 28672 data: 5 4 4 4 5 4 4
117182024-02-27T21:48:30.576Z30/30 Write block 213 len 8192 data: 10 10
117192024-02-27T21:48:30.576Z----------------------------------------------------------------
117202024-02-27T21:48:30.577Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
117212024-02-27T21:48:30.577ZStates: Active Active Active
117222024-02-27T21:48:30.577ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
117232024-02-27T21:48:30.577Z 1 Acked 1000 Write 5 Done Done Done false
117242024-02-27T21:48:30.577Z 2 Acked 1001 Write 7 Done Done Done false
117252024-02-27T21:48:30.577Z 3 Acked 1002 Read 6 Done Done Done false
117262024-02-27T21:48:30.577Z 4 Acked 1003 Read 6 Done Done Done false
117272024-02-27T21:48:30.577Z 5 Acked 1004 Write 4 Done Done Done false
117282024-02-27T21:48:30.577Z 6 Acked 1005 Write 9 Done Done Done false
117292024-02-27T21:48:30.577Z 7 Acked 1006 Read 6 Sent Done Done false
117302024-02-27T21:48:30.578Z 8 Acked 1007 Write 5 Sent Done Done false
117312024-02-27T21:48:30.578Z 9 Acked 1008 Write 10 Sent Done Done false
117322024-02-27T21:48:30.578Z 10 Acked 1009 Read 8 Sent Done Done false
117332024-02-27T21:48:30.578Z 11 Acked 1010 Write 3 Sent Done Done false
117342024-02-27T21:48:30.578Z 12 Acked 1011 Read 5 Sent Done Done false
117352024-02-27T21:48:30.578Z 13 Acked 1012 Write 10 Sent Done Done false
117362024-02-27T21:48:30.578Z 14 Acked 1013 Write 7 Sent Done Done false
117372024-02-27T21:48:30.578Z 15 Acked 1014 Read 5 Sent Done Done false
117382024-02-27T21:48:30.579Z 16 Acked 1015 Read 10 Sent Done Done false
117392024-02-27T21:48:30.579Z 17 Acked 1016 Write 8 Sent Done Done false
117402024-02-27T21:48:30.579Z 18 Acked 1017 Read 10 Sent Done Done false
117412024-02-27T21:48:30.579Z 19 Acked 1018 Read 3 Sent Done Done false
117422024-02-27T21:48:30.579Z 20 Acked 1019 Read 3 Sent Done Done false
117432024-02-27T21:48:30.579Z 21 Acked 1020 Write 3 Sent Done Done false
117442024-02-27T21:48:30.579Z 22 Acked 1021 Read 6 Sent Done Done false
117452024-02-27T21:48:30.579Z 23 Acked 1022 Read 10 Sent Done Done false
117462024-02-27T21:48:30.579Z 24 Acked 1023 Write 2 Sent Done Done false
117472024-02-27T21:48:30.579Z 25 Acked 1024 Write 8 Sent Done Done false
117482024-02-27T21:48:30.580Z 26 Acked 1025 Read 3 Sent Done Done false
117492024-02-27T21:48:30.580Z 27 Acked 1026 Read 10 Sent Done Done false
117502024-02-27T21:48:30.580Z 28 Acked 1027 Write 6 Sent Done Done false
117512024-02-27T21:48:30.580Z 29 Acked 1028 Write 7 Sent Sent Sent false
117522024-02-27T21:48:30.580Z 30 Acked 1029 Write 2 Sent Sent Sent false
117532024-02-27T21:48:30.580Z STATES DS:0 DS:1 DS:2 TOTAL
117542024-02-27T21:48:30.580Z New 0 0 0 0
117552024-02-27T21:48:30.580Z Sent 24 2 2 28
117562024-02-27T21:48:30.580Z Done 6 28 28 62
117572024-02-27T21:48:30.580Z Skipped 0 0 0 0
117582024-02-27T21:48:30.581Z Error 0 0 0 0
117592024-02-27T21:48:30.581ZLast Flush: 0 0 0
117602024-02-27T21:48:30.581ZDownstairs last five completed:
117612024-02-27T21:48:30.581ZUpstairs last five completed: 30 29 28 27 26
117622024-02-27T21:48:30.581Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:28.941158303Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1576}
117632024-02-27T21:48:30.581ZWrote out file "/tmp/test_repair_verify.data"
117642024-02-27T21:48:30.581Z
117652024-02-27T21:48:30.581ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117662024-02-27T21:48:30.590ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117672024-02-27T21:48:30.590Z 0 000-019 104 104 104 196 196 196 F T T <---
117682024-02-27T21:48:30.590Z 1 020-039 104 104 104 196 196 196 F F F
117692024-02-27T21:48:30.591Z 2 040-059 106 106 106 198 198 198 F F F
117702024-02-27T21:48:30.591Z 3 060-079 106 106 106 198 198 198 T T T
117712024-02-27T21:48:30.595Z 4 080-099 107 107 107 202 202 202 F T T <---
117722024-02-27T21:48:30.595Z 5 100-119 99 99 99 191 191 191 F T T <---
117732024-02-27T21:48:30.595Z 6 120-139 104 104 104 196 196 196 F T T <---
117742024-02-27T21:48:30.595Z 7 140-159 97 97 97 189 189 189 F T T <---
117752024-02-27T21:48:30.596Z 8 160-179 104 104 104 196 196 196 F T T <---
117762024-02-27T21:48:30.596Z 9 180-199 102 102 102 192 192 192 F T T <---
117772024-02-27T21:48:30.596Z 10 200-219 106 106 106 201 201 201 T T T
117782024-02-27T21:48:30.596Z 11 220-239 107 107 107 202 202 202 T T T
117792024-02-27T21:48:30.596Z 12 240-259 102 102 102 193 193 193 F T T <---
117802024-02-27T21:48:30.596Z 13 260-279 104 104 104 196 196 196 T T T
117812024-02-27T21:48:30.596Z 14 280-299 102 102 102 194 194 194 F F F
117822024-02-27T21:48:30.596Z 15 300-319 102 102 102 193 193 193 F F F
117832024-02-27T21:48:30.597Z 16 320-339 106 106 106 200 200 200 F T T <---
117842024-02-27T21:48:30.597Z 17 340-359 104 104 104 196 196 196 F T T <---
117852024-02-27T21:48:30.597Z 18 360-379 107 107 107 202 202 202 F F F
117862024-02-27T21:48:30.597Z 19 380-399 102 102 102 192 192 192 F F F
117872024-02-27T21:48:30.597Z 20 400-419 99 99 99 191 191 191 F T T <---
117882024-02-27T21:48:30.597Z 21 420-439 106 106 106 201 201 201 F F F
117892024-02-27T21:48:30.598Z 22 440-459 104 104 104 196 196 196 F F F
117902024-02-27T21:48:30.598Z 23 460-479 106 106 106 198 198 198 F F F
117912024-02-27T21:48:30.598Z 24 480-499 106 106 106 199 199 199 T T T
117922024-02-27T21:48:30.598Z 25 500-519 105 105 105 197 197 197 F F F
117932024-02-27T21:48:30.598Z 26 520-539 104 104 104 196 196 196 F T T <---
117942024-02-27T21:48:30.598Z 27 540-559 107 107 107 202 202 202 F F F
117952024-02-27T21:48:30.599Z 28 560-579 104 104 104 196 196 196 F F F
117962024-02-27T21:48:30.599Z 29 580-599 106 106 106 200 200 200 F F F
117972024-02-27T21:48:30.599ZMax gen: 107, Max flush: 202
117982024-02-27T21:48:30.599ZError: Difference in extent metadata found!
117992024-02-27T21:48:30.599ZA Difference in extent metadata is expected here
118002024-02-27T21:48:30.599ZOn loop 54
118012024-02-27T21:48:30.599Z
118022024-02-27T21:48:30.605ZVerifying data now
118032024-02-27T21:48:36.489ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
118042024-02-27T21:48:36.494Z202a203
118052024-02-27T21:48:36.494Z> 7,
118062024-02-27T21:48:36.494Z206d206
118072024-02-27T21:48:36.494Z< 5,
118082024-02-27T21:48:36.495Z215a216,217
118092024-02-27T21:48:36.495Z> 10,
118102024-02-27T21:48:36.495Z> 10,
118112024-02-27T21:48:36.495Z217,218d218
118122024-02-27T21:48:36.495Z< 9,
118132024-02-27T21:48:36.495Z< 9,
118142024-02-27T21:48:36.495Z263c263
118152024-02-27T21:48:36.495Z< 9,
118162024-02-27T21:48:36.496Z---
118172024-02-27T21:48:36.496Z> 10,
118182024-02-27T21:48:36.496Zdiff found after verify
118192024-02-27T21:48:36.500ZLoop: 54 Downstairs dump after verify (and repair):
118202024-02-27T21:48:36.510ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118212024-02-27T21:48:36.510Z 0 000-019 108 108 108 203 203 203 F F F
118222024-02-27T21:48:36.515Z 1 020-039 104 104 104 196 196 196 F F F
118232024-02-27T21:48:36.515Z 2 040-059 106 106 106 198 198 198 F F F
118242024-02-27T21:48:36.515Z 3 060-079 108 108 108 203 203 203 F F F
118252024-02-27T21:48:36.515Z 4 080-099 108 108 108 203 203 203 F F F
118262024-02-27T21:48:36.515Z 5 100-119 108 108 108 203 203 203 F F F
118272024-02-27T21:48:36.515Z 6 120-139 108 108 108 203 203 203 F F F
118282024-02-27T21:48:36.516Z 7 140-159 108 108 108 203 203 203 F F F
118292024-02-27T21:48:36.516Z 8 160-179 108 108 108 203 203 203 F F F
118302024-02-27T21:48:36.516Z 9 180-199 108 108 108 203 203 203 F F F
118312024-02-27T21:48:36.516Z 10 200-219 108 108 108 203 203 203 F F F
118322024-02-27T21:48:36.516Z 11 220-239 108 108 108 203 203 203 F F F
118332024-02-27T21:48:36.516Z 12 240-259 108 108 108 203 203 203 F F F
118342024-02-27T21:48:36.516Z 13 260-279 108 108 108 203 203 203 F F F
118352024-02-27T21:48:36.516Z 14 280-299 102 102 102 194 194 194 F F F
118362024-02-27T21:48:36.517Z 15 300-319 102 102 102 193 193 193 F F F
118372024-02-27T21:48:36.517Z 16 320-339 108 108 108 203 203 203 F F F
118382024-02-27T21:48:36.517Z 17 340-359 108 108 108 203 203 203 F F F
118392024-02-27T21:48:36.517Z 18 360-379 107 107 107 202 202 202 F F F
118402024-02-27T21:48:36.517Z 19 380-399 102 102 102 192 192 192 F F F
118412024-02-27T21:48:36.517Z 20 400-419 108 108 108 203 203 203 F F F
118422024-02-27T21:48:36.517Z 21 420-439 106 106 106 201 201 201 F F F
118432024-02-27T21:48:36.518Z 22 440-459 104 104 104 196 196 196 F F F
118442024-02-27T21:48:36.518Z 23 460-479 106 106 106 198 198 198 F F F
118452024-02-27T21:48:36.518Z 24 480-499 108 108 108 203 203 203 F F F
118462024-02-27T21:48:36.518Z 25 500-519 105 105 105 197 197 197 F F F
118472024-02-27T21:48:36.518Z 26 520-539 108 108 108 203 203 203 F F F
118482024-02-27T21:48:36.518Z 27 540-559 107 107 107 202 202 202 F F F
118492024-02-27T21:48:36.519Z 28 560-579 104 104 104 196 196 196 F F F
118502024-02-27T21:48:36.519Z 29 580-599 106 106 106 200 200 200 F F F
118512024-02-27T21:48:36.519ZMax gen: 108, Max flush: 203
118522024-02-27T21:48:36.519Z
118532024-02-27T21:48:36.519ZBegin loop 55
118542024-02-27T21:48:36.519ZDownstairs to restart: 1
118552024-02-27T21:48:36.535Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.915464797Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587}
118562024-02-27T21:48:36.535Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.915788309Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587}
118572024-02-27T21:48:36.542Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.915811366Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587}
118582024-02-27T21:48:36.542Z{"msg":"Crucible 9355a803-7d18-414c-acb7-1dfb7f9a9de0 has session id: 4e32eb9c-3182-468f-ae7a-d710af581256","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.915846342Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118592024-02-27T21:48:36.542Z{"msg":"Upstairs opts: Upstairs UUID: 9355a803-7d18-414c-acb7-1dfb7f9a9de0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.915894606Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118602024-02-27T21:48:36.542Z{"msg":"Crucible stats registered with UUID: 9355a803-7d18-414c-acb7-1dfb7f9a9de0","v":0,"name":"crucible","level":30Crucible runtime is spawned
118612024-02-27T21:48:36.542Z,"time":"2024-02-27T21:48:34.915939291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118622024-02-27T21:48:36.543Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916049558Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587}
118632024-02-27T21:48:36.543Z{"msg":"9355a803-7d18-414c-acb7-1dfb7f9a9de0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916139738Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118642024-02-27T21:48:36.543Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916176733Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118652024-02-27T21:48:36.543Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916215989Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118662024-02-27T21:48:36.543Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916241206Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118672024-02-27T21:48:36.543Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916275162Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118682024-02-27T21:48:36.543Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916298379Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118692024-02-27T21:48:36.543Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916332795Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118702024-02-27T21:48:36.544Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916349893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118712024-02-27T21:48:36.544Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916385938Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118722024-02-27T21:48:36.544Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916413395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118732024-02-27T21:48:36.544Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.91645417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118742024-02-27T21:48:36.544Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.916578746Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118752024-02-27T21:48:36.544Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.91662881Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118762024-02-27T21:48:36.544Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.917133041Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118772024-02-27T21:48:36.544Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.917170086Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118782024-02-27T21:48:36.544Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.917185984Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118792024-02-27T21:48:36.544Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.917205182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118802024-02-27T21:48:36.545Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.917241218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118812024-02-27T21:48:36.545Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.917280533Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118822024-02-27T21:48:36.545Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.917798382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118832024-02-27T21:48:36.545Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.917973722Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118842024-02-27T21:48:36.545Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918118105Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118852024-02-27T21:48:36.545Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918315691Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118862024-02-27T21:48:36.545Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918350677Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118872024-02-27T21:48:36.546Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918366145Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118882024-02-27T21:48:36.546Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918499109Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118892024-02-27T21:48:36.546Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918551063Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118902024-02-27T21:48:36.546Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918575691Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118912024-02-27T21:48:36.546Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918648272Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118922024-02-27T21:48:36.546Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918675569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118932024-02-27T21:48:36.546Z{"msg":"[0]R flush_numbers[0..12]: [203, 196, 198, 203, 203, 203, 203, 203, 203, 203, 203, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918690517Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118942024-02-27T21:48:36.546Z{"msg":"[0]R generation[0..12]: [108, 104, 106, 108, 108, 108, 108, 108, 108, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918712894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118952024-02-27T21:48:36.546Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918727893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118962024-02-27T21:48:36.547Z{"msg":"[1]R flush_numbers[0..12]: [203, 196, 198, 203, 203, 203, 203, 203, 203, 203, 203, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.91874914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118972024-02-27T21:48:36.547Z{"msg":"[1]R generation[0..12]: [108, 104, 106, 108, 108, 108, 108, 108, 108, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918763578Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118982024-02-27T21:48:36.547Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918805944Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
118992024-02-27T21:48:36.547Z{"msg":"[2]R flush_numbers[0..12]: [203, 196, 198, 203, 203, 203, 203, 203, 203, 203, 203, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918820602Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119002024-02-27T21:48:36.547Z{Wait for a query_work_queue command to finish before sending IO
119012024-02-27T21:48:36.547Z"msg":"[2]R generation[0..12]: [108, 104, 106, 108, 108, 108, 108, 108, 108, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.91883787Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119022024-02-27T21:48:36.547Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918859277Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119032024-02-27T21:48:36.547Z{"msg":"Max found gen is 109","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918873336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119042024-02-27T21:48:36.547Z{"msg":"Generation requested: 110 >= found:109","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918894123Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119052024-02-27T21:48:36.547Z{"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918907472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119062024-02-27T21:48:36.548Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918927279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"":"downstairs","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119072024-02-27T21:48:36.548Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918941157Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119082024-02-27T21:48:36.548Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918959965Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"0","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119092024-02-27T21:48:36.548Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918974284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"1","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119102024-02-27T21:48:36.548Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.918994141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"client":"2","":"downstairs","session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119112024-02-27T21:48:36.548Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.91900772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119122024-02-27T21:48:36.548Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.919031347Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119132024-02-27T21:48:36.548Z{"msg":"9355a803-7d18-414c-acb7-1dfb7f9a9de0 is now active with session: 4e32eb9c-3182-468f-ae7a-d710af581256","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.919045765Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119142024-02-27T21:48:36.549Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.919065473Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587,"session_id":"4e32eb9c-3182-468f-ae7a-d710af581256"}
119152024-02-27T21:48:36.549Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:34.919078231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587}
119162024-02-27T21:48:36.549ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
119172024-02-27T21:48:36.555ZLoading write count information from file "/tmp/test_repair_verify.data"
119182024-02-27T21:48:36.555ZRun Repair workload
119192024-02-27T21:48:36.555Z01/30 Write block 269 len 16384 data: 8 8 9 10
119202024-02-27T21:48:36.555Z02/30 Write block 244 len 40960 data: 7 8 9 9 8 8 8 7 9 9
119212024-02-27T21:48:36.555Z03/30 Write block 119 len 12288 data: 8 10 11
119222024-02-27T21:48:36.555Z04/30 Read block 91 len 36864
119232024-02-27T21:48:36.555Z05/30 Read block 431 len 16384
119242024-02-27T21:48:36.555Z06/30 Write block 315 len 36864 data: 8 11 11 11 13 12 10 8 8
119252024-02-27T21:48:36.556Z07/30 Flush
119262024-02-27T21:48:36.556Z08/30 Write block 73 len 40960 data: 6 7 8 7 8 7 7 6 6 6
119272024-02-27T21:48:36.556Z09/30 Write block 174 len 40960 data: 11 10 12 11 13 11 14 16 16 13
119282024-02-27T21:48:36.556Z10/30 Write block 588 len 36864 data: 8 10 9 7 8 8 7 6 7
119292024-02-27T21:48:36.556Z11/30 Read block 351 len 28672
119302024-02-27T21:48:36.556Z12/30 Read block 453 len 12288
119312024-02-27T21:48:36.560Z13/30 Write block 247 len 20480 data: 10 9 9 9 8
119322024-02-27T21:48:36.560Z14/30 Read block 344 len 24576
119332024-02-27T21:48:36.561Z15/30 Write block 316 len 8192 data: 12 12
119342024-02-27T21:48:36.561Z16/30 Write block 27 len 16384 data: 8 7 8 10
119352024-02-27T21:48:36.561Z17/30 Flush
119362024-02-27T21:48:36.564Z18/30 Flush
119372024-02-27T21:48:36.565Z19/30 Write block 458 len 20480 data: 3 7 6 6 6
119382024-02-27T21:48:36.565Z20/30 Read block 367 len 32768
119392024-02-27T21:48:36.568Z21/30 Write block 540 len 40960 data: 6 6 7 6 5 6 7 7 12 12
119402024-02-27T21:48:36.569Z22/30 Write block 558 len 16384 data: 14 12 13 14
119412024-02-27T21:48:36.569Z23/30 Read block 261 len 4096
119422024-02-27T21:48:36.569Z24/30 Write block 244 len 32768 data: 8 9 10 11 10 10 10 9
119432024-02-27T21:48:36.573Z25/30 Read block 298 len 12288
119442024-02-27T21:48:36.573Z26/30 Write block 214 len 40960 data: 10 10 11 14 14 12 11 10 10 9
119452024-02-27T21:48:36.573Z27/30 Write block 49 len 24576 data: 8 9 8 9 9 8
119462024-02-27T21:48:36.573Z28/30 Write block 36 len 40960 data: 9 6 6 8 9 9 9 11 11 13
119472024-02-27T21:48:36.573Z29/30 Read block 97 len 16384
119482024-02-27T21:48:36.578Z30/30 Write block 316 len 28672 data: 13 13 12 14 13 11 9
119492024-02-27T21:48:36.578Z----------------------------------------------------------------
119502024-02-27T21:48:36.578Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:12
119512024-02-27T21:48:36.578ZStates: Active Active Active
119522024-02-27T21:48:36.578ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119532024-02-27T21:48:36.579Z 19 Acked 1018 Write 5 Done Sent Done false
119542024-02-27T21:48:36.579Z 20 Acked 1019 Read 8 Done Sent Done false
119552024-02-27T21:48:36.579Z 21 Acked 1020 Write 10 Done Sent Done false
119562024-02-27T21:48:36.579Z 22 Acked 1021 Write 4 Done Sent Done false
119572024-02-27T21:48:36.579Z 23 Acked 1022 Read 1 Done Sent Done false
119582024-02-27T21:48:36.579Z 24 Acked 1023 Write 8 Done Sent Done false
119592024-02-27T21:48:36.580Z 25 Acked 1024 Read 3 Done Sent Done false
119602024-02-27T21:48:36.580Z 26 Acked 1025 Write 10 Done Sent Done false
119612024-02-27T21:48:36.580Z 27 Acked 1026 Write 6 Done Sent Done false
119622024-02-27T21:48:36.580Z 28 Acked 1027 Write 10 Done Sent Done false
119632024-02-27T21:48:36.580Z 29 Acked 1028 Read 4 Done Sent Done false
119642024-02-27T21:48:36.580Z 30 Acked 1029 Write 7 Sent Sent Sent false
119652024-02-27T21:48:36.580Z STATES DS:0 DS:1 DS:2 TOTAL
119662024-02-27T21:48:36.581Z New 0 0 0 0
119672024-02-27T21:48:36.581Z Sent 1 12 1 14
119682024-02-27T21:48:36.581Z Done 11 0 11 22
119692024-02-27T21:48:36.581Z Skipped 0 0 0 0
119702024-02-27T21:48:36.581Z Error 0 0 0 0
119712024-02-27T21:48:36.581ZLast Flush: 1017 1017 1017
119722024-02-27T21:48:36.581ZDownstairs last five completed: 1017 1016 1015 1014 1013
119732024-02-27T21:48:36.581ZUpstairs last five completed: 30 29 28 27 26
119742024-02-27T21:48:36.581Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:34.955473404Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1587}
119752024-02-27T21:48:36.582ZWrote out file "/tmp/test_repair_verify.data"
119762024-02-27T21:48:36.586Z
119772024-02-27T21:48:36.586ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119782024-02-27T21:48:36.606ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119792024-02-27T21:48:36.606Z 0 000-019 108 108 108 203 203 203 F F F
119802024-02-27T21:48:36.606Z 1 020-039 110 110 110 205 205 205 T F T <---
119812024-02-27T21:48:36.606Z 2 040-059 106 106 106 198 198 198 T F T <---
119822024-02-27T21:48:36.606Z 3 060-079 110 110 110 205 205 205 F F F
119832024-02-27T21:48:36.606Z 4 080-099 110 110 110 205 205 205 F F F
119842024-02-27T21:48:36.606Z 5 100-119 110 110 110 204 204 204 F F F
119852024-02-27T21:48:36.606Z 6 120-139 110 110 110 204 204 204 F F F
119862024-02-27T21:48:36.607Z 7 140-159 108 108 108 203 203 203 F F F
119872024-02-27T21:48:36.607Z 8 160-179 110 110 110 205 205 205 F F F
119882024-02-27T21:48:36.611Z 9 180-199 110 110 110 205 205 205 F F F
119892024-02-27T21:48:36.611Z 10 200-219 108 108 108 203 203 203 T F T <---
119902024-02-27T21:48:36.611Z 11 220-239 108 108 108 203 203 203 T F T <---
119912024-02-27T21:48:36.611Z 12 240-259 110 110 110 205 205 205 T F T <---
119922024-02-27T21:48:36.611Z 13 260-279 110 110 110 204 204 204 F F F
119932024-02-27T21:48:36.611Z 14 280-299 102 102 102 194 194 194 F F F
119942024-02-27T21:48:36.612Z 15 300-319 110 110 110 205 205 205 T F T <---
119952024-02-27T21:48:36.612Z 16 320-339 110 110 110 204 204 204 T F T <---
119962024-02-27T21:48:36.612Z 17 340-359 108 108 108 203 203 203 F F F
119972024-02-27T21:48:36.612Z 18 360-379 107 107 107 202 202 202 F F F
119982024-02-27T21:48:36.612Z 19 380-399 102 102 102 192 192 192 F F F
119992024-02-27T21:48:36.612Z 20 400-419 108 108 108 203 203 203 F F F
120002024-02-27T21:48:36.612Z 21 420-439 106 106 106 201 201 201 F F F
120012024-02-27T21:48:36.612Z 22 440-459 104 104 104 196 196 196 T F T <---
120022024-02-27T21:48:36.613Z 23 460-479 106 106 106 198 198 198 T F T <---
120032024-02-27T21:48:36.613Z 24 480-499 108 108 108 203 203 203 F F F
120042024-02-27T21:48:36.613Z 25 500-519 105 105 105 197 197 197 F F F
120052024-02-27T21:48:36.613Z 26 520-539 108 108 108 203 203 203 F F F
120062024-02-27T21:48:36.613Z 27 540-559 107 107 107 202 202 202 T F T <---
120072024-02-27T21:48:36.613Z 28 560-579 104 104 104 196 196 196 T F T <---
120082024-02-27T21:48:36.613Z 29 580-599 110 110 110 205 205 205 F F F
120092024-02-27T21:48:36.613ZMax gen: 110, Max flush: 205
120102024-02-27T21:48:36.614ZError: Difference in extent metadata found!
120112024-02-27T21:48:36.614ZA Difference in extent metadata is expected here
120122024-02-27T21:48:36.614ZOn loop 55
120132024-02-27T21:48:36.614Z
120142024-02-27T21:48:36.614ZVerifying data now
120152024-02-27T21:48:41.197ZNo change after verify
120162024-02-27T21:48:41.197ZLoop: 55 Downstairs dump after verify (and repair):
120172024-02-27T21:48:41.210ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120182024-02-27T21:48:41.211Z 0 000-019 108 108 108 203 203 203 F F F
120192024-02-27T21:48:41.211Z 1 020-039 111 111 111 206 206 206 F F F
120202024-02-27T21:48:41.211Z 2 040-059 111 111 111 206 206 206 F F F
120212024-02-27T21:48:41.211Z 3 060-079 110 110 110 205 205 205 F F F
120222024-02-27T21:48:41.211Z 4 080-099 110 110 110 205 205 205 F F F
120232024-02-27T21:48:41.211Z 5 100-119 110 110 110 204 204 204 F F F
120242024-02-27T21:48:41.215Z 6 120-139 110 110 110 204 204 204 F F F
120252024-02-27T21:48:41.216Z 7 140-159 108 108 108 203 203 203 F F F
120262024-02-27T21:48:41.216Z 8 160-179 110 110 110 205 205 205 F F F
120272024-02-27T21:48:41.216Z 9 180-199 110 110 110 205 205 205 F F F
120282024-02-27T21:48:41.216Z 10 200-219 111 111 111 206 206 206 F F F
120292024-02-27T21:48:41.216Z 11 220-239 111 111 111 206 206 206 F F F
120302024-02-27T21:48:41.216Z 12 240-259 111 111 111 206 206 206 F F F
120312024-02-27T21:48:41.216Z 13 260-279 110 110 110 204 204 204 F F F
120322024-02-27T21:48:41.216Z 14 280-299 102 102 102 194 194 194 F F F
120332024-02-27T21:48:41.217Z 15 300-319 111 111 111 206 206 206 F F F
120342024-02-27T21:48:41.217Z 16 320-339 111 111 111 206 206 206 F F F
120352024-02-27T21:48:41.217Z 17 340-359 108 108 108 203 203 203 F F F
120362024-02-27T21:48:41.217Z 18 360-379 107 107 107 202 202 202 F F F
120372024-02-27T21:48:41.217Z 19 380-399 102 102 102 192 192 192 F F F
120382024-02-27T21:48:41.217Z 20 400-419 108 108 108 203 203 203 F F F
120392024-02-27T21:48:41.218Z 21 420-439 106 106 106 201 201 201 F F F
120402024-02-27T21:48:41.218Z 22 440-459 111 111 111 206 206 206 F F F
120412024-02-27T21:48:41.218Z 23 460-479 111 111 111 206 206 206 F F F
120422024-02-27T21:48:41.218Z 24 480-499 108 108 108 203 203 203 F F F
120432024-02-27T21:48:41.219Z 25 500-519 105 105 105 197 197 197 F F F
120442024-02-27T21:48:41.219Z 26 520-539 108 108 108 203 203 203 F F F
120452024-02-27T21:48:41.219Z 27 540-559 111 111 111 206 206 206 F F F
120462024-02-27T21:48:41.219Z 28 560-579 111 111 111 206 206 206 F F F
120472024-02-27T21:48:41.219Z 29 580-599 110 110 110 205 205 205 F F F
120482024-02-27T21:48:41.219ZMax gen: 111, Max flush: 206
120492024-02-27T21:48:41.219Z
120502024-02-27T21:48:41.219ZBegin loop 56
120512024-02-27T21:48:41.219ZDownstairs to restart: 1
120522024-02-27T21:48:41.232Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.61165798Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595}
120532024-02-27T21:48:41.240Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.611961825Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595}
120542024-02-27T21:48:41.240Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612010579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595}
120552024-02-27T21:48:41.240Z{"msg":"Crucible 12b93bca-2af2-40e5-96be-a11e18e4e6b0 has session id: eced4244-d1b3-4048-ac46-b3e7730761ea","v":0,"name":"crucible","level":30Crucible runtime is spawned
120562024-02-27T21:48:41.240Z,"time":"2024-02-27T21:48:39.612049004Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120572024-02-27T21:48:41.240Z{"msg":"Upstairs opts: Upstairs UUID: 12b93bca-2af2-40e5-96be-a11e18e4e6b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612119716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120582024-02-27T21:48:41.241Z{"msg":"Crucible stats registered with UUID: 12b93bca-2af2-40e5-96be-a11e18e4e6b0","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612162281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120592024-02-27T21:48:41.241Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612180449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595}
120602024-02-27T21:48:41.241Z{"msg":"12b93bca-2af2-40e5-96be-a11e18e4e6b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612223484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120612024-02-27T21:48:41.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612248741Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120622024-02-27T21:48:41.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612284607Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120632024-02-27T21:48:41.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612342829Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120642024-02-27T21:48:41.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612389164Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120652024-02-27T21:48:41.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.61242177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120662024-02-27T21:48:41.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612463225Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120672024-02-27T21:48:41.242Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612487312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120682024-02-27T21:48:41.242Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612514779Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120692024-02-27T21:48:41.242Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612552675Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120702024-02-27T21:48:41.242Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612586201Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120712024-02-27T21:48:41.242Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612622177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120722024-02-27T21:48:41.242Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.612653133Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120732024-02-27T21:48:41.242Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.613327973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120742024-02-27T21:48:41.242Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.6133573Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120752024-02-27T21:48:41.243Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.613372618Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120762024-02-27T21:48:41.243Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.613388596Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120772024-02-27T21:48:41.243Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.613423612Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120782024-02-27T21:48:41.243Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.613447579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120792024-02-27T21:48:41.243Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.613930213Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120802024-02-27T21:48:41.243Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.613974307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120812024-02-27T21:48:41.243Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614004264Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120822024-02-27T21:48:41.243Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614487467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120832024-02-27T21:48:41.243Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.61454177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120842024-02-27T21:48:41.243Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614566338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120852024-02-27T21:48:41.243Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614589335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120862024-02-27T21:48:41.243Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614603943Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120872024-02-27T21:48:41.243Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614618152Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120882024-02-27T21:48:41.244Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.6146306Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120892024-02-27T21:48:41.244Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614644818Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120902024-02-27T21:48:41.244Z{"msg":"[0]R flush_numbers[0..12]: [203, 206, 206, 205, 205, 204, 204, 203, 205, 205, 206, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614658887Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120912024-02-27T21:48:41.244Z{"msg":"[0]R generation[0..12]: [108, 111, 111, 110, 110, 110, 110, 108, 110, 110, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614673065Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120922024-02-27T21:48:41.244Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614695092Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120932024-02-27T21:48:41.244Z{"msg":"[1]R flush_numbers[0..12]: [203, 206, 206, 205, 205, 204, 204, 203, 205, 205, 206, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614709911Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120942024-02-27T21:48:41.244Z{"msg":"[1]R generation[0..12]: [108, 111, 111, 110, 110, 110, 110, 108, 110, 110, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614723709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120952024-02-27T21:48:41.244Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614744397Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120962024-02-27T21:48:41.244Z{"msg":"[2]R flush_numbers[0..12]: [203, 206, 206, 205, 205, 204, 204, 203, 205, 205, 206, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614758955Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120972024-02-27T21:48:41.244Z{"msg":"[2]R generation[0..12]: [108, 111, 111, 110, 110, 110, 110, 108, 110, 110, 111, 111]","v":0,Wait for a query_work_queue command to finish before sending IO
120982024-02-27T21:48:41.244Z"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614776103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
120992024-02-27T21:48:41.244Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614791251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121002024-02-27T21:48:41.244Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614814338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121012024-02-27T21:48:41.244Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614843555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121022024-02-27T21:48:41.244Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614858583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121032024-02-27T21:48:41.244Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.61488816Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"":"downstairs","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121042024-02-27T21:48:41.244Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614912777Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121052024-02-27T21:48:41.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614940014Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"0","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121062024-02-27T21:48:41.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614964161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"1","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121072024-02-27T21:48:41.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.614984138Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"client":"2","":"downstairs","session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121082024-02-27T21:48:41.244Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.615032823Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121092024-02-27T21:48:41.244Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.6150592Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121102024-02-27T21:48:41.245ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
121112024-02-27T21:48:41.245Z{"msg":"12b93bca-2af2-40e5-96be-a11e18e4e6b0 is now active with session: eced4244-d1b3-4048-ac46-b3e7730761ea","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.615095655Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121122024-02-27T21:48:41.245Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.615116303Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595,"session_id":"eced4244-d1b3-4048-ac46-b3e7730761ea"}
121132024-02-27T21:48:41.245Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:39.615130231Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595}
121142024-02-27T21:48:41.245ZLoading write count information from file "/tmp/test_repair_verify.data"
121152024-02-27T21:48:41.245ZRun Repair workload
121162024-02-27T21:48:41.245Z01/30 Write block 424 len 20480 data: 5 5 5 6 7
121172024-02-27T21:48:41.250Z02/30 Read block 52 len 24576
121182024-02-27T21:48:41.250Z03/30 Read block 538 len 12288
121192024-02-27T21:48:41.250Z04/30 Flush
121202024-02-27T21:48:41.250Z05/30 Write block 402 len 16384 data: 8 7 8 7
121212024-02-27T21:48:41.250Z06/30 Write block 523 len 24576 data: 5 6 5 5 5 6
121222024-02-27T21:48:41.250Z07/30 Read block 587 len 16384
121232024-02-27T21:48:41.251Z08/30 Write block 40 len 32768 data: 10 10 10 12 12 14 9 9
121242024-02-27T21:48:41.251Z09/30 Write block 215 len 8192 data: 11 12
121252024-02-27T21:48:41.251Z10/30 Read block 294 len 8192
121262024-02-27T21:48:41.251Z11/30 Write block 258 len 32768 data: 11 10 10 7 7 6 7 7
121272024-02-27T21:48:41.251Z12/30 Read block 441 len 4096
121282024-02-27T21:48:41.251Z13/30 Read block 531 len 36864
121292024-02-27T21:48:41.251Z14/30 Write block 348 len 16384 data: 10 11 11 10
121302024-02-27T21:48:41.251Z15/30 Write block 358 len 4096 data: 7
121312024-02-27T21:48:41.273Z16/30 Read block 238 len 20480
121322024-02-27T21:48:41.273Z17/30 Read block 25 len 12288
121332024-02-27T21:48:41.274Z18/30 Read block 169 len 4096
121342024-02-27T21:48:41.274Z19/30 Write block 457 len 32768 data: 2 4 8 7 7 7 6 6
121352024-02-27T21:48:41.274Z20/30 Write block 188 len 40960 data: 12 12 10 10 9 7 7 5 5 5
121362024-02-27T21:48:41.274Z21/30 Write block 87 len 36864 data: 9 10 11 8 9 5 8 9 8
121372024-02-27T21:48:41.274Z22/30 Read block 69 len 36864
121382024-02-27T21:48:41.278Z23/30 Read block 461 len 12288
121392024-02-27T21:48:41.279Z24/30 Read block 581 len 4096
121402024-02-27T21:48:41.279Z25/30 Write block 580 len 12288 data: 7 7 10
121412024-02-27T21:48:41.279Z26/30 Write block 517 len 8192 data: 12 8
121422024-02-27T21:48:41.279Z27/30 Write block 161 len 16384 data: 9 8 8 7
121432024-02-27T21:48:41.279Z28/30 Write block 355 len 8192 data: 12 11
121442024-02-27T21:48:41.279Z29/30 Read block 419 len 40960
121452024-02-27T21:48:41.279Z30/30 Read block 333 len 36864
121462024-02-27T21:48:41.279Z----------------------------------------------------------------
121472024-02-27T21:48:41.279Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:26
121482024-02-27T21:48:41.279ZStates: Active Active Active
121492024-02-27T21:48:41.279ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121502024-02-27T21:48:41.280Z 5 Acked 1004 Write 4 Done Sent Done false
121512024-02-27T21:48:41.280Z 6 Acked 1005 Write 6 Done Sent Done false
121522024-02-27T21:48:41.280Z 7 Acked 1006 Read 4 Done Sent Done false
121532024-02-27T21:48:41.280Z 8 Acked 1007 Write 8 Done Sent Done false
121542024-02-27T21:48:41.280Z 9 Acked 1008 Write 2 Done Sent Done false
121552024-02-27T21:48:41.280Z 10 Acked 1009 Read 2 Done Sent Done false
121562024-02-27T21:48:41.280Z 11 Acked 1010 Write 8 Done Sent Done false
121572024-02-27T21:48:41.280Z 12 Acked 1011 Read 1 Done Sent Done false
121582024-02-27T21:48:41.280Z 13 Acked 1012 Read 9 Done Sent Done false
121592024-02-27T21:48:41.280Z 14 Acked 1013 Write 4 Done Sent Done false
121602024-02-27T21:48:41.281Z 15 Acked 1014 Write 1 Done Sent Done false
121612024-02-27T21:48:41.281Z 16 Acked 1015 Read 5 Done Sent Done false
121622024-02-27T21:48:41.281Z 17 Acked 1016 Read 3 Done Sent Done false
121632024-02-27T21:48:41.281Z 18 Acked 1017 Read 1 Done Sent Done false
121642024-02-27T21:48:41.281Z 19 Acked 1018 Write 8 Done Sent Done false
121652024-02-27T21:48:41.281Z 20 Acked 1019 Write 10 Done Sent Done false
121662024-02-27T21:48:41.281Z 21 Acked 1020 Write 9 Done Sent Done false
121672024-02-27T21:48:41.281Z 22 Acked 1021 Read 9 Done Sent Done false
121682024-02-27T21:48:41.281Z 23 Acked 1022 Read 3 Done Sent Done false
121692024-02-27T21:48:41.281Z 24 Acked 1023 Read 1 Done Sent Done false
121702024-02-27T21:48:41.281Z 25 Acked 1024 Write 3 Done Sent Done false
121712024-02-27T21:48:41.282Z 26 Acked 1025 Write 2 Done Sent Done false
121722024-02-27T21:48:41.282Z 27 Acked 1026 Write 4 Done Sent Done false
121732024-02-27T21:48:41.282Z 28 Acked 1027 Write 2 Done Sent Done false
121742024-02-27T21:48:41.282Z 29 Acked 1028 Read 10 Done Sent Done false
121752024-02-27T21:48:41.282Z 30 Acked 1029 Read 9 Sent Sent Done false
121762024-02-27T21:48:41.282Z STATES DS:0 DS:1 DS:2 TOTAL
121772024-02-27T21:48:41.282Z New 0 0 0 0
121782024-02-27T21:48:41.282Z Sent 1 26 0 27
121792024-02-27T21:48:41.282Z Done 25 0 26 51
121802024-02-27T21:48:41.282Z Skipped 0 0 0 0
121812024-02-27T21:48:41.283Z Error 0 0 0 0
121822024-02-27T21:48:41.283ZLast Flush: 1003 1003 1003
121832024-02-27T21:48:41.283ZDownstairs last five completed: 1003 1002 1001 1000
121842024-02-27T21:48:41.283ZUpstairs last five completed: 30 29 28 27 26
121852024-02-27T21:48:41.283Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:39.641662236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1595}
121862024-02-27T21:48:41.283ZWrote out file "/tmp/test_repair_verify.data"
121872024-02-27T21:48:41.283Z
121882024-02-27T21:48:41.283ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121892024-02-27T21:48:41.289ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121902024-02-27T21:48:41.289Z 0 000-019 108 108 108 203 203 203 F F F
121912024-02-27T21:48:41.289Z 1 020-039 111 111 111 206 206 206 F F F
121922024-02-27T21:48:41.289Z 2 040-059 111 111 111 206 206 206 T F T <---
121932024-02-27T21:48:41.289Z 3 060-079 110 110 110 205 205 205 F F F
121942024-02-27T21:48:41.289Z 4 080-099 110 110 110 205 205 205 T F T <---
121952024-02-27T21:48:41.289Z 5 100-119 110 110 110 204 204 204 F F F
121962024-02-27T21:48:41.289Z 6 120-139 110 110 110 204 204 204 F F F
121972024-02-27T21:48:41.289Z 7 140-159 108 108 108 203 203 203 F F F
121982024-02-27T21:48:41.289Z 8 160-179 110 110 110 205 205 205 T F T <---
121992024-02-27T21:48:41.290Z 9 180-199 110 110 110 205 205 205 T F T <---
122002024-02-27T21:48:41.294Z 10 200-219 111 111 111 206 206 206 T F T <---
122012024-02-27T21:48:41.294Z 11 220-239 111 111 111 206 206 206 F F F
122022024-02-27T21:48:41.294Z 12 240-259 111 111 111 206 206 206 T F T <---
122032024-02-27T21:48:41.294Z 13 260-279 110 110 110 204 204 204 T F T <---
122042024-02-27T21:48:41.294Z 14 280-299 102 102 102 194 194 194 F F F
122052024-02-27T21:48:41.294Z 15 300-319 111 111 111 206 206 206 F F F
122062024-02-27T21:48:41.294Z 16 320-339 111 111 111 206 206 206 F F F
122072024-02-27T21:48:41.295Z 17 340-359 108 108 108 203 203 203 T F T <---
122082024-02-27T21:48:41.295Z 18 360-379 107 107 107 202 202 202 F F F
122092024-02-27T21:48:41.295Z 19 380-399 102 102 102 192 192 192 F F F
122102024-02-27T21:48:41.295Z 20 400-419 108 108 108 203 203 203 T F T <---
122112024-02-27T21:48:41.295Z 21 420-439 112 112 112 207 207 207 F F F
122122024-02-27T21:48:41.295Z 22 440-459 111 111 111 206 206 206 T F T <---
122132024-02-27T21:48:41.295Z 23 460-479 111 111 111 206 206 206 T F T <---
122142024-02-27T21:48:41.295Z 24 480-499 108 108 108 203 203 203 F F F
122152024-02-27T21:48:41.296Z 25 500-519 105 105 105 197 197 197 T F T <---
122162024-02-27T21:48:41.296Z 26 520-539 108 108 108 203 203 203 T F T <---
122172024-02-27T21:48:41.296Z 27 540-559 111 111 111 206 206 206 F F F
122182024-02-27T21:48:41.296Z 28 560-579 111 111 111 206 206 206 F F F
122192024-02-27T21:48:41.296Z 29 580-599 110 110 110 205 205 205 T F T <---
122202024-02-27T21:48:41.296ZMax gen: 112, Max flush: 207
122212024-02-27T21:48:41.296ZError: Difference in extent metadata found!
122222024-02-27T21:48:41.296ZA Difference in extent metadata is expected here
122232024-02-27T21:48:41.296ZOn loop 56
122242024-02-27T21:48:41.297Z
122252024-02-27T21:48:41.301ZVerifying data now
122262024-02-27T21:48:46.005ZNo change after verify
122272024-02-27T21:48:46.005ZLoop: 56 Downstairs dump after verify (and repair):
122282024-02-27T21:48:46.019ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122292024-02-27T21:48:46.019Z 0 000-019 108 108 108 203 203 203 F F F
122302024-02-27T21:48:46.020Z 1 020-039 111 111 111 206 206 206 F F F
122312024-02-27T21:48:46.020Z 2 040-059 113 113 113 208 208 208 F F F
122322024-02-27T21:48:46.020Z 3 060-079 110 110 110 205 205 205 F F F
122332024-02-27T21:48:46.020Z 4 080-099 113 113 113 208 208 208 F F F
122342024-02-27T21:48:46.020Z 5 100-119 110 110 110 204 204 204 F F F
122352024-02-27T21:48:46.020Z 6 120-139 110 110 110 204 204 204 F F F
122362024-02-27T21:48:46.020Z 7 140-159 108 108 108 203 203 203 F F F
122372024-02-27T21:48:46.020Z 8 160-179 113 113 113 208 208 208 F F F
122382024-02-27T21:48:46.028Z 9 180-199 113 113 113 208 208 208 F F F
122392024-02-27T21:48:46.028Z 10 200-219 113 113 113 208 208 208 F F F
122402024-02-27T21:48:46.028Z 11 220-239 111 111 111 206 206 206 F F F
122412024-02-27T21:48:46.028Z 12 240-259 113 113 113 208 208 208 F F F
122422024-02-27T21:48:46.028Z 13 260-279 113 113 113 208 208 208 F F F
122432024-02-27T21:48:46.028Z 14 280-299 102 102 102 194 194 194 F F F
122442024-02-27T21:48:46.028Z 15 300-319 111 111 111 206 206 206 F F F
122452024-02-27T21:48:46.028Z 16 320-339 111 111 111 206 206 206 F F F
122462024-02-27T21:48:46.028Z 17 340-359 113 113 113 208 208 208 F F F
122472024-02-27T21:48:46.028Z 18 360-379 107 107 107 202 202 202 F F F
122482024-02-27T21:48:46.028Z 19 380-399 102 102 102 192 192 192 F F F
122492024-02-27T21:48:46.028Z 20 400-419 113 113 113 208 208 208 F F F
122502024-02-27T21:48:46.029Z 21 420-439 112 112 112 207 207 207 F F F
122512024-02-27T21:48:46.029Z 22 440-459 113 113 113 208 208 208 F F F
122522024-02-27T21:48:46.029Z 23 460-479 113 113 113 208 208 208 F F F
122532024-02-27T21:48:46.029Z 24 480-499 108 108 108 203 203 203 F F F
122542024-02-27T21:48:46.029Z 25 500-519 113 113 113 208 208 208 F F F
122552024-02-27T21:48:46.029Z 26 520-539 113 113 113 208 208 208 F F F
122562024-02-27T21:48:46.029Z 27 540-559 111 111 111 206 206 206 F F F
122572024-02-27T21:48:46.029Z 28 560-579 111 111 111 206 206 206 F F F
122582024-02-27T21:48:46.029Z 29 580-599 113 113 113 208 208 208 F F F
122592024-02-27T21:48:46.029ZMax gen: 113, Max flush: 208
122602024-02-27T21:48:46.029Z
122612024-02-27T21:48:46.029ZBegin loop 57
122622024-02-27T21:48:46.029ZDownstairs to restart: 2
122632024-02-27T21:48:46.043Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.422375415Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603}
122642024-02-27T21:48:46.043Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
122652024-02-27T21:48:46.043Z,"time":"2024-02-27T21:48:44.422721254Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603}
122662024-02-27T21:48:46.048Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.422763639Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603}
122672024-02-27T21:48:46.048Z{"msg":"Crucible 59af1676-4526-4476-921c-eb6ec8a4316f has session id: 724cd26f-3718-49a7-8d1a-95dc923dd188","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.422817453Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122682024-02-27T21:48:46.048Z{"msg":"Upstairs opts: Upstairs UUID: 59af1676-4526-4476-921c-eb6ec8a4316f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.422874436Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122692024-02-27T21:48:46.048Z{"msg":"Crucible stats registered with UUID: 59af1676-4526-4476-921c-eb6ec8a4316f","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.422920161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122702024-02-27T21:48:46.048Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.422948158Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603}
122712024-02-27T21:48:46.048Z{"msg":"59af1676-4526-4476-921c-eb6ec8a4316f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.422974025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122722024-02-27T21:48:46.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423027008Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122732024-02-27T21:48:46.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423075942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122742024-02-27T21:48:46.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423127307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122752024-02-27T21:48:46.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423209877Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122762024-02-27T21:48:46.049Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.42327249Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122772024-02-27T21:48:46.049Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423315684Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122782024-02-27T21:48:46.049Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423342681Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122792024-02-27T21:48:46.049Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423384266Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122802024-02-27T21:48:46.049Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423412613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122812024-02-27T21:48:46.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423451218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122822024-02-27T21:48:46.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423486934Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122832024-02-27T21:48:46.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.423528629Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122842024-02-27T21:48:46.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.424137888Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122852024-02-27T21:48:46.049Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.424174113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122862024-02-27T21:48:46.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.424190981Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122872024-02-27T21:48:46.049Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.424277991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122882024-02-27T21:48:46.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.424437502Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122892024-02-27T21:48:46.049Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.424467319Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122902024-02-27T21:48:46.049Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.42488352Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122912024-02-27T21:48:46.049Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.424940243Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122922024-02-27T21:48:46.049Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425194993Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122932024-02-27T21:48:46.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425396409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122942024-02-27T21:48:46.049Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425465741Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122952024-02-27T21:48:46.049Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425503357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122962024-02-27T21:48:46.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425518075Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122972024-02-27T21:48:46.050Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425545712Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122982024-02-27T21:48:46.050Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425572008Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
122992024-02-27T21:48:46.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425684565Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123002024-02-27T21:48:46.050Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425745018Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123012024-02-27T21:48:46.050Z{"msg":"[0]R flush_numbers[0..12]: [203, 206, 208, 205, 208, 204, 204, 203, 208, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425764436Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123022024-02-27T21:48:46.050Z{"msg":"[0]R generation[0..12]: [108, 111, 113, 110, 113, 110, 110, 108, 113, 113, 113, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425787543Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123032024-02-27T21:48:46.050Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425802861Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123042024-02-27T21:48:46.050Z{"msg":"[1]R flush_numbers[0..12]: [203, 206, 208, 205, 208, 204, 204, 203, 208, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425830288Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123052024-02-27T21:48:46.050Z{"msg":"[1]R generation[0..12]: [108, 111, 113, 110, 113, 110, 110, 108, 113, 113, 113, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425853895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123062024-02-27T21:48:46.054Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425869453Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123072024-02-27T21:48:46.054Z{"msg":"[2]R flush_numbers[0..12]: [203, 206, 208, 205, 208, 204, 204, 203, 208, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.42589769Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123082024-02-27T21:48:46.055Z{"msg":"[2]R generation[0..12]: [108, 111, 113, 110, 113, 110, 110, 108, 113, 113, 113, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425915508Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123092024-02-27T21:48:46.055Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.425938256Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123102024-02-27T21:48:46.055Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
123112024-02-27T21:48:46.055Z,"time":"2024-02-27T21:48:44.425952164Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123122024-02-27T21:48:46.055Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426022706Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123132024-02-27T21:48:46.055Z{"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426039653Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123142024-02-27T21:48:46.055Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426061421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"":"downstairs","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123152024-02-27T21:48:46.055Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426076889Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123162024-02-27T21:48:46.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426089627Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"0","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123172024-02-27T21:48:46.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426111095Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"1","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123182024-02-27T21:48:46.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426132913Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"client":"2","":"downstairs","session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123192024-02-27T21:48:46.055Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426160479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123202024-02-27T21:48:46.055Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426177467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123212024-02-27T21:48:46.055Z{"msg":"59af1676-4526-4476-921c-eb6ec8a4316f is now active with session: 724cd26f-3718-49a7-8d1a-95dc923dd188","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426191515Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123222024-02-27T21:48:46.055Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426203684Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603,"session_id":"724cd26f-3718-49a7-8d1a-95dc923dd188"}
123232024-02-27T21:48:46.055Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:44.426224572Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603}
123242024-02-27T21:48:46.055ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123252024-02-27T21:48:46.055ZLoading write count information from file "/tmp/test_repair_verify.data"
123262024-02-27T21:48:46.055ZRun Repair workload
123272024-02-27T21:48:46.055Z01/30 Write block 520 len 12288 data: 8 5 7
123282024-02-27T21:48:46.055Z02/30 Read block 256 len 24576
123292024-02-27T21:48:46.055Z03/30 Flush
123302024-02-27T21:48:46.055Z04/30 Flush
123312024-02-27T21:48:46.055Z05/30 Write block 413 len 32768 data: 7 7 8 9 12 11 10 6
123322024-02-27T21:48:46.056Z06/30 Flush
123332024-02-27T21:48:46.059Z07/30 Write block 274 len 4096 data: 8
123342024-02-27T21:48:46.059Z08/30 Write block 514 len 4096 data: 12
123352024-02-27T21:48:46.059Z09/30 Write block 12 len 36864 data: 14 13 12 13 13 12 10 9 9
123362024-02-27T21:48:46.059Z10/30 Write block 348 len 20480 data: 11 12 12 11 9
123372024-02-27T21:48:46.059Z11/30 Read block 542 len 28672
123382024-02-27T21:48:46.062Z12/30 Write block 414 len 20480 data: 8 9 10 13 12
123392024-02-27T21:48:46.062Z13/30 Read block 357 len 4096
123402024-02-27T21:48:46.062Z14/30 Write block 124 len 20480 data: 13 10 10 10 11
123412024-02-27T21:48:46.062Z15/30 Write block 553 len 36864 data: 11 12 11 11 17 15 13 14 15
123422024-02-27T21:48:46.066Z16/30 Write block 422 len 12288 data: 5 5 6
123432024-02-27T21:48:46.066Z17/30 Write block 286 len 8192 data: 6 7
123442024-02-27T21:48:46.066Z18/30 Read block 117 len 36864
123452024-02-27T21:48:46.066Z19/30 Write block 393 len 32768 data: 10 8 8 8 11 10 11 12
123462024-02-27T21:48:46.069Z20/30 Write block 72 len 40960 data: 9 7 8 9 8 9 8 8 7 7
123472024-02-27T21:48:46.069Z21/30 Write block 241 len 36864 data: 6 5 6 9 10 11 12 11 11
123482024-02-27T21:48:46.069Z22/30 Read block 460 len 20480
123492024-02-27T21:48:46.069Z23/30 Read block 110 len 12288
123502024-02-27T21:48:46.072Z24/30 Write block 565 len 20480 data: 6 5 4 7 8
123512024-02-27T21:48:46.072Z25/30 Read block 263 len 8192
123522024-02-27T21:48:46.072Z26/30 Read block 350 len 8192
123532024-02-27T21:48:46.073Z27/30 Read block 464 len 16384
123542024-02-27T21:48:46.076Z28/30 Write block 232 len 32768 data: 16 15 15 15 15 9 9 9
123552024-02-27T21:48:46.076Z29/30 Read block 583 len 20480
123562024-02-27T21:48:46.076Z30/30 Write block 324 len 4096 data: 7
123572024-02-27T21:48:46.076Z----------------------------------------------------------------
123582024-02-27T21:48:46.076Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
123592024-02-27T21:48:46.076ZStates: Active Active Active
123602024-02-27T21:48:46.076ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123612024-02-27T21:48:46.076Z 1 Acked 1000 Write 3 Done Done Done false
123622024-02-27T21:48:46.076Z 2 Acked 1001 Read 6 Done Done Done false
123632024-02-27T21:48:46.077Z 3 Acked 1002 Flush 0 Done Done Sent false
123642024-02-27T21:48:46.077Z 4 Acked 1003 Flush 0 Done Done Sent false
123652024-02-27T21:48:46.077Z 5 Acked 1004 Write 8 Done Done Sent false
123662024-02-27T21:48:46.077Z 6 Acked 1005 Flush 0 Done Done Sent false
123672024-02-27T21:48:46.077Z 7 Acked 1006 Write 1 Done Done Sent false
123682024-02-27T21:48:46.077Z 8 Acked 1007 Write 1 Done Done Sent false
123692024-02-27T21:48:46.077Z 9 Acked 1008 Write 9 Done Done Sent false
123702024-02-27T21:48:46.077Z 10 Acked 1009 Write 5 Done Done Sent false
123712024-02-27T21:48:46.077Z 11 Acked 1010 Read 7 Done Done Sent false
123722024-02-27T21:48:46.077Z 12 Acked 1011 Write 5 Done Done Sent false
123732024-02-27T21:48:46.077Z 13 Acked 1012 Read 1 Done Done Sent false
123742024-02-27T21:48:46.077Z 14 Acked 1013 Write 5 Done Done Sent false
123752024-02-27T21:48:46.077Z 15 Acked 1014 Write 9 Done Done Sent false
123762024-02-27T21:48:46.077Z 16 Acked 1015 Write 3 Done Done Sent false
123772024-02-27T21:48:46.077Z 17 Acked 1016 Write 2 Done Done Sent false
123782024-02-27T21:48:46.077Z 18 Acked 1017 Read 9 Done Done Sent false
123792024-02-27T21:48:46.077Z 19 Acked 1018 Write 8 Done Done Sent false
123802024-02-27T21:48:46.077Z 20 Acked 1019 Write 10 Done Done Sent false
123812024-02-27T21:48:46.077Z 21 Acked 1020 Write 9 Done Done Sent false
123822024-02-27T21:48:46.077Z 22 Acked 1021 Read 5 Done Done Sent false
123832024-02-27T21:48:46.077Z 23 Acked 1022 Read 3 Done Done Sent false
123842024-02-27T21:48:46.077Z 24 Acked 1023 Write 5 Done Done Sent false
123852024-02-27T21:48:46.077Z 25 Acked 1024 Read 2 Done Done Sent false
123862024-02-27T21:48:46.077Z 26 Acked 1025 Read 2 Done Done Sent false
123872024-02-27T21:48:46.078Z 27 Acked 1026 Read 4 Done Done Sent false
123882024-02-27T21:48:46.078Z 28 Acked 1027 Write 8 Done Done Sent false
123892024-02-27T21:48:46.078Z 29 Acked 1028 Read 5 Done Done Sent false
123902024-02-27T21:48:46.078Z 30 Acked 1029 Write 1 Sent Sent Sent false
123912024-02-27T21:48:46.078Z STATES DS:0 DS:1 DS:2 TOTAL
123922024-02-27T21:48:46.078Z New 0 0 0 0
123932024-02-27T21:48:46.078Z Sent 1 1 28 30
123942024-02-27T21:48:46.078Z Done 29 29 2 60
123952024-02-27T21:48:46.078Z Skipped 0 0 0 0
123962024-02-27T21:48:46.078Z Error 0 0 0 0
123972024-02-27T21:48:46.078ZLast Flush: 1005 1005 0
123982024-02-27T21:48:46.078ZDownstairs last five completed:
123992024-02-27T21:48:46.078ZUpstairs last five completed: 30 29 28 27 26
124002024-02-27T21:48:46.078Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:44.455675313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1603}
124012024-02-27T21:48:46.081ZWrote out file "/tmp/test_repair_verify.data"
124022024-02-27T21:48:46.081Z
124032024-02-27T21:48:46.085ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
124042024-02-27T21:48:46.107ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124052024-02-27T21:48:46.107Z 0 000-019 108 108 108 203 203 203 T T F <---
124062024-02-27T21:48:46.107Z 1 020-039 111 111 111 206 206 206 T T F <---
124072024-02-27T21:48:46.107Z 2 040-059 113 113 113 208 208 208 F F F
124082024-02-27T21:48:46.107Z 3 060-079 110 110 110 205 205 205 T T F <---
124092024-02-27T21:48:46.107Z 4 080-099 113 113 113 208 208 208 T T F <---
124102024-02-27T21:48:46.107Z 5 100-119 110 110 110 204 204 204 F F F
124112024-02-27T21:48:46.107Z 6 120-139 110 110 110 204 204 204 T T F <---
124122024-02-27T21:48:46.107Z 7 140-159 108 108 108 203 203 203 F F F
124132024-02-27T21:48:46.107Z 8 160-179 113 113 113 208 208 208 F F F
124142024-02-27T21:48:46.107Z 9 180-199 113 113 113 208 208 208 F F F
124152024-02-27T21:48:46.107Z 10 200-219 113 113 113 208 208 208 F F F
124162024-02-27T21:48:46.107Z 11 220-239 111 111 111 206 206 206 T T F <---
124172024-02-27T21:48:46.107Z 12 240-259 113 113 113 208 208 208 T T F <---
124182024-02-27T21:48:46.111Z 13 260-279 113 113 113 208 208 208 T T F <---
124192024-02-27T21:48:46.111Z 14 280-299 102 102 102 194 194 194 T T F <---
124202024-02-27T21:48:46.111Z 15 300-319 111 111 111 206 206 206 F F F
124212024-02-27T21:48:46.111Z 16 320-339 111 111 111 206 206 206 T T F <---
124222024-02-27T21:48:46.111Z 17 340-359 113 113 113 208 208 208 T T F <---
124232024-02-27T21:48:46.111Z 18 360-379 107 107 107 202 202 202 F F F
124242024-02-27T21:48:46.111Z 19 380-399 102 102 102 192 192 192 T T F <---
124252024-02-27T21:48:46.111Z 20 400-419 114 114 113 211 211 208 T T F <---
124262024-02-27T21:48:46.111Z 21 420-439 114 114 112 211 211 207 T T F <---
124272024-02-27T21:48:46.111Z 22 440-459 113 113 113 208 208 208 F F F
124282024-02-27T21:48:46.111Z 23 460-479 113 113 113 208 208 208 F F F
124292024-02-27T21:48:46.111Z 24 480-499 108 108 108 203 203 203 F F F
124302024-02-27T21:48:46.111Z 25 500-519 113 113 113 208 208 208 T T F <---
124312024-02-27T21:48:46.111Z 26 520-539 114 114 113 209 209 208 F F T <---
124322024-02-27T21:48:46.111Z 27 540-559 111 111 111 206 206 206 T T F <---
124332024-02-27T21:48:46.111Z 28 560-579 111 111 111 206 206 206 T T F <---
124342024-02-27T21:48:46.111Z 29 580-599 113 113 113 208 208 208 F F F
124352024-02-27T21:48:46.112ZMax gen: 114, Max flush: 211
124362024-02-27T21:48:46.112ZError: Difference in extent metadata found!
124372024-02-27T21:48:46.112ZA Difference in extent metadata is expected here
124382024-02-27T21:48:46.112ZOn loop 57
124392024-02-27T21:48:46.112Z
124402024-02-27T21:48:46.117ZVerifying data now
124412024-02-27T21:48:52.007ZNo change after verify
124422024-02-27T21:48:52.007ZLoop: 57 Downstairs dump after verify (and repair):
124432024-02-27T21:48:52.022ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124442024-02-27T21:48:52.022Z 0 000-019 115 115 115 212 212 212 F F F
124452024-02-27T21:48:52.022Z 1 020-039 115 115 115 212 212 212 F F F
124462024-02-27T21:48:52.022Z 2 040-059 113 113 113 208 208 208 F F F
124472024-02-27T21:48:52.022Z 3 060-079 115 115 115 212 212 212 F F F
124482024-02-27T21:48:52.022Z 4 080-099 115 115 115 212 212 212 F F F
124492024-02-27T21:48:52.026Z 5 100-119 110 110 110 204 204 204 F F F
124502024-02-27T21:48:52.026Z 6 120-139 115 115 115 212 212 212 F F F
124512024-02-27T21:48:52.026Z 7 140-159 108 108 108 203 203 203 F F F
124522024-02-27T21:48:52.026Z 8 160-179 113 113 113 208 208 208 F F F
124532024-02-27T21:48:52.026Z 9 180-199 113 113 113 208 208 208 F F F
124542024-02-27T21:48:52.026Z 10 200-219 113 113 113 208 208 208 F F F
124552024-02-27T21:48:52.026Z 11 220-239 115 115 115 212 212 212 F F F
124562024-02-27T21:48:52.026Z 12 240-259 115 115 115 212 212 212 F F F
124572024-02-27T21:48:52.026Z 13 260-279 115 115 115 212 212 212 F F F
124582024-02-27T21:48:52.026Z 14 280-299 115 115 115 212 212 212 F F F
124592024-02-27T21:48:52.026Z 15 300-319 111 111 111 206 206 206 F F F
124602024-02-27T21:48:52.026Z 16 320-339 115 115 115 212 212 212 F F F
124612024-02-27T21:48:52.026Z 17 340-359 115 115 115 212 212 212 F F F
124622024-02-27T21:48:52.026Z 18 360-379 107 107 107 202 202 202 F F F
124632024-02-27T21:48:52.026Z 19 380-399 115 115 115 212 212 212 F F F
124642024-02-27T21:48:52.026Z 20 400-419 115 115 115 212 212 212 F F F
124652024-02-27T21:48:52.026Z 21 420-439 115 115 115 212 212 212 F F F
124662024-02-27T21:48:52.026Z 22 440-459 113 113 113 208 208 208 F F F
124672024-02-27T21:48:52.026Z 23 460-479 113 113 113 208 208 208 F F F
124682024-02-27T21:48:52.026Z 24 480-499 108 108 108 203 203 203 F F F
124692024-02-27T21:48:52.026Z 25 500-519 115 115 115 212 212 212 F F F
124702024-02-27T21:48:52.026Z 26 520-539 114 114 114 209 209 209 F F F
124712024-02-27T21:48:52.027Z 27 540-559 115 115 115 212 212 212 F F F
124722024-02-27T21:48:52.027Z 28 560-579 115 115 115 212 212 212 F F F
124732024-02-27T21:48:52.027Z 29 580-599 113 113 113 208 208 208 F F F
124742024-02-27T21:48:52.027ZMax gen: 115, Max flush: 212
124752024-02-27T21:48:52.027Z
124762024-02-27T21:48:52.027ZBegin loop 58
124772024-02-27T21:48:52.027ZDownstairs to restart: 2
124782024-02-27T21:48:52.043Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.421671304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611}
124792024-02-27T21:48:52.043Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.421977018Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611}
124802024-02-27T21:48:52.043Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"Crucible runtime is spawned
124812024-02-27T21:48:52.043Zcrucible","level":30,"time":"2024-02-27T21:48:50.422021953Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611}
124822024-02-27T21:48:52.050Z{"msg":"Crucible 5ff00cdb-06b0-4c35-a699-f1d24688762d has session id: 31fa1cf7-2a3f-4c12-ba74-71952f7077b4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422063958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124832024-02-27T21:48:52.050Z{"msg":"Upstairs opts: Upstairs UUID: 5ff00cdb-06b0-4c35-a699-f1d24688762d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422121051Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124842024-02-27T21:48:52.050Z{"msg":"Crucible stats registered with UUID: 5ff00cdb-06b0-4c35-a699-f1d24688762d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422164516Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124852024-02-27T21:48:52.050Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422204881Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611}
124862024-02-27T21:48:52.050Z{"msg":"5ff00cdb-06b0-4c35-a699-f1d24688762d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422307069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124872024-02-27T21:48:52.050Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422348634Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124882024-02-27T21:48:52.050Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422397959Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124892024-02-27T21:48:52.050Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422426635Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124902024-02-27T21:48:52.050Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422452472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124912024-02-27T21:48:52.050Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422485228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124922024-02-27T21:48:52.050Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422622222Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124932024-02-27T21:48:52.050Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422730129Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124942024-02-27T21:48:52.050Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422764845Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124952024-02-27T21:48:52.050Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.42280926Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124962024-02-27T21:48:52.050Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422839227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124972024-02-27T21:48:52.050Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422878192Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124982024-02-27T21:48:52.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.422903649Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
124992024-02-27T21:48:52.051Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.423509717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125002024-02-27T21:48:52.051Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.423545893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125012024-02-27T21:48:52.051Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.4235701Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125022024-02-27T21:48:52.051Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.423640932Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125032024-02-27T21:48:52.051Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.423688107Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125042024-02-27T21:48:52.051Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.423734611Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125052024-02-27T21:48:52.051Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424174279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125062024-02-27T21:48:52.051Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424219134Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125072024-02-27T21:48:52.051Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424383265Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125082024-02-27T21:48:52.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424743882Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125092024-02-27T21:48:52.051Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424820433Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125102024-02-27T21:48:52.051Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424869177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125112024-02-27T21:48:52.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424885175Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125122024-02-27T21:48:52.051Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424908323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125132024-02-27T21:48:52.051Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424922811Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125142024-02-27T21:48:52.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424935589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125152024-02-27T21:48:52.051Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424957457Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125162024-02-27T21:48:52.051Z{"msg":"[0]R flush_numbers[0..12]: [212, 212, 208, 212, 212, 204, 212, 203, 208, 208, 208, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424972045Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125172024-02-27T21:48:52.051Z{"msg":"[0]R generation[0..12]: [115, 115, 113, 115, 115, 110, 115, 108, 113, 113, 113, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.424992933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125182024-02-27T21:48:52.051Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425019839Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125192024-02-27T21:48:52.051Z{"msg":"[1]R flush_numbers[0..12]: [212, 212, 208, 212, 212, 204, 212, 203, 208, 208, 208, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425042187Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125202024-02-27T21:48:52.051Z{"msg":"[1]R generation[0..12]: [115, 115, 113, 115, 115, 110, 115, 108, 113, 113, 113, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425056235Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125212024-02-27T21:48:52.051Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425075483Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125222024-02-27T21:48:52.051Z{"msg":"[2]R flush_numbers[0..12]: [212, 212, 208, 212, 212, 204, 212, 203, 208, 208, 208, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425089921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125232024-02-27T21:48:52.052Z{"msg":"[2]R generation[0..12]: [115, 115, 113, 115, 115, 110, 115, 108, 113, 113, 113, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425111449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125242024-02-27T21:48:52.052Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425126987Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125252024-02-27T21:48:52.052ZWait for a query_work_queue command to finish before sending IO
125262024-02-27T21:48:52.052Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425146945Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125272024-02-27T21:48:52.052Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425168172Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125282024-02-27T21:48:52.052Z{"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.42518202Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125292024-02-27T21:48:52.052Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425195979Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"":"downstairs","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125302024-02-27T21:48:52.052Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425216176Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125312024-02-27T21:48:52.052Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425228605Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"0","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125322024-02-27T21:48:52.052Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425248903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"1","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125332024-02-27T21:48:52.052Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425262351Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"client":"2","":"downstairs","session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125342024-02-27T21:48:52.052Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.42527582Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125352024-02-27T21:48:52.052Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425295447Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125362024-02-27T21:48:52.052Z{"msg":"5ff00cdb-06b0-4c35-a699-f1d24688762d is now active with session: 31fa1cf7-2a3f-4c12-ba74-71952f7077b4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425308945Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125372024-02-27T21:48:52.052Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425328113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611,"session_id":"31fa1cf7-2a3f-4c12-ba74-71952f7077b4"}
125382024-02-27T21:48:52.052Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:50.425340692Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611}
125392024-02-27T21:48:52.052ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125402024-02-27T21:48:52.056ZLoading write count information from file "/tmp/test_repair_verify.data"
125412024-02-27T21:48:52.056ZRun Repair workload
125422024-02-27T21:48:52.056Z01/30 Write block 584 len 20480 data: 12 11 11 9 9
125432024-02-27T21:48:52.056Z02/30 Read block 320 len 28672
125442024-02-27T21:48:52.056Z03/30 Read block 9 len 28672
125452024-02-27T21:48:52.056Z04/30 Read block 529 len 28672
125462024-02-27T21:48:52.056Z05/30 Read block 509 len 24576
125472024-02-27T21:48:52.056Z06/30 Write block 169 len 16384 data: 6 7 6 6
125482024-02-27T21:48:52.056Z07/30 Write block 107 len 12288 data: 13 12 11
125492024-02-27T21:48:52.056Z08/30 Read block 386 len 32768
125502024-02-27T21:48:52.056Z09/30 Write block 18 len 12288 data: 11 10 10
125512024-02-27T21:48:52.056Z10/30 Read block 295 len 12288
125522024-02-27T21:48:52.059Z11/30 Write block 95 len 24576 data: 9 8 7 6 9 10
125532024-02-27T21:48:52.060Z12/30 Flush
125542024-02-27T21:48:52.065Z13/30 Write block 391 len 8192 data: 11 9
125552024-02-27T21:48:52.066Z14/30 Flush
125562024-02-27T21:48:52.066Z15/30 Write block 421 len 8192 data: 5 6
125572024-02-27T21:48:52.069Z16/30 Read block 56 len 40960
125582024-02-27T21:48:52.069Z17/30 Read block 81 len 4096
125592024-02-27T21:48:52.069Z18/30 Write block 390 len 16384 data: 9 12 10 11
125602024-02-27T21:48:52.069Z19/30 Read block 302 len 32768
125612024-02-27T21:48:52.069Z20/30 Write block 68 len 20480 data: 7 8 7 7 10
125622024-02-27T21:48:52.069Z21/30 Read block 531 len 16384
125632024-02-27T21:48:52.069Z22/30 Read block 316 len 8192
125642024-02-27T21:48:52.069Z23/30 Flush
125652024-02-27T21:48:52.072Z24/30 Write block 46 len 40960 data: 10 10 9 9 10 9 10 10 9 6
125662024-02-27T21:48:52.072Z25/30 Write block 230 len 4096 data: 12
125672024-02-27T21:48:52.075Z26/30 Read block 534 len 40960
125682024-02-27T21:48:52.075Z27/30 Read block 524 len 12288
125692024-02-27T21:48:52.075Z28/30 Read block 317 len 8192
125702024-02-27T21:48:52.079Z29/30 Write block 70 len 28672 data: 8 8 11 8 9 10 9
125712024-02-27T21:48:52.079Z30/30 Read block 381 len 36864
125722024-02-27T21:48:52.079Z----------------------------------------------------------------
125732024-02-27T21:48:52.079Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
125742024-02-27T21:48:52.079ZStates: Active Active Active
125752024-02-27T21:48:52.079ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125762024-02-27T21:48:52.079Z 1 Acked 1000 Write 5 Done Done Sent false
125772024-02-27T21:48:52.079Z 2 Acked 1001 Read 7 Done Done Sent false
125782024-02-27T21:48:52.079Z 3 Acked 1002 Read 7 Done Done Sent false
125792024-02-27T21:48:52.079Z 4 Acked 1003 Read 7 Done Done Sent false
125802024-02-27T21:48:52.079Z 5 Acked 1004 Read 6 Done Done Sent false
125812024-02-27T21:48:52.079Z 6 Acked 1005 Write 4 Done Done Sent false
125822024-02-27T21:48:52.079Z 7 Acked 1006 Write 3 Done Done Sent false
125832024-02-27T21:48:52.079Z 8 Acked 1007 Read 8 Done Done Sent false
125842024-02-27T21:48:52.079Z 9 Acked 1008 Write 3 Done Done Sent false
125852024-02-27T21:48:52.079Z 10 Acked 1009 Read 3 Done Done Sent false
125862024-02-27T21:48:52.080Z 11 Acked 1010 Write 6 Done Done Sent false
125872024-02-27T21:48:52.080Z 12 Acked 1011 Flush 0 Done Done Sent false
125882024-02-27T21:48:52.080Z 13 Acked 1012 Write 2 Done Done Sent false
125892024-02-27T21:48:52.080Z 14 Acked 1013 Flush 0 Done Done Sent false
125902024-02-27T21:48:52.080Z 15 Acked 1014 Write 2 Done Done Sent false
125912024-02-27T21:48:52.080Z 16 Acked 1015 Read 10 Done Done Sent false
125922024-02-27T21:48:52.080Z 17 Acked 1016 Read 1 Done Done Sent false
125932024-02-27T21:48:52.080Z 18 Acked 1017 Write 4 Done Done Sent false
125942024-02-27T21:48:52.080Z 19 Acked 1018 Read 8 Done Done Sent false
125952024-02-27T21:48:52.080Z 20 Acked 1019 Write 5 Done Done Sent false
125962024-02-27T21:48:52.080Z 21 Acked 1020 Read 4 Done Done Sent false
125972024-02-27T21:48:52.080Z 22 Acked 1021 Read 2 Done Done Sent false
125982024-02-27T21:48:52.080Z 23 Acked 1022 Flush 0 Done Done Sent false
125992024-02-27T21:48:52.080Z 24 Acked 1023 Write 10 Done Done Sent false
126002024-02-27T21:48:52.080Z 25 Acked 1024 Write 1 Done Done Sent false
126012024-02-27T21:48:52.080Z 26 Acked 1025 Read 10 Done Done Sent false
126022024-02-27T21:48:52.080Z 27 Acked 1026 Read 3 Done Done Sent false
126032024-02-27T21:48:52.080Z 28 Acked 1027 Read 2 Done Done Sent false
126042024-02-27T21:48:52.080Z 29 Acked 1028 Write 7 Done Done Sent false
126052024-02-27T21:48:52.080Z 30 Acked 1029 Read 9 Done Sent Sent false
126062024-02-27T21:48:52.080Z STATES DS:0 DS:1 DS:2 TOTAL
126072024-02-27T21:48:52.080Z New 0 0 0 0
126082024-02-27T21:48:52.080Z Sent 0 1 30 31
126092024-02-27T21:48:52.080Z Done 30 29 0 59
126102024-02-27T21:48:52.080Z Skipped 0 0 0 0
126112024-02-27T21:48:52.081Z Error 0 0 0 0
126122024-02-27T21:48:52.081ZLast Flush: 1022 1022 0
126132024-02-27T21:48:52.081ZDownstairs last five completed:
126142024-02-27T21:48:52.081ZUpstairs last five completed: 30 29 28 27 26
126152024-02-27T21:48:52.081Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:50.456374266Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1611}
126162024-02-27T21:48:52.081ZWrote out file "/tmp/test_repair_verify.data"
126172024-02-27T21:48:52.084Z
126182024-02-27T21:48:52.087ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
126192024-02-27T21:48:52.101ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126202024-02-27T21:48:52.101Z 0 000-019 116 116 115 213 213 212 F F F <---
126212024-02-27T21:48:52.101Z 1 020-039 116 116 115 213 213 212 F F F <---
126222024-02-27T21:48:52.101Z 2 040-059 113 113 113 208 208 208 T T F <---
126232024-02-27T21:48:52.101Z 3 060-079 116 116 115 215 215 212 T T F <---
126242024-02-27T21:48:52.101Z 4 080-099 116 116 115 213 213 212 F F F <---
126252024-02-27T21:48:52.101Z 5 100-119 116 116 110 213 213 204 F F F <---
126262024-02-27T21:48:52.101Z 6 120-139 115 115 115 212 212 212 F F F
126272024-02-27T21:48:52.102Z 7 140-159 108 108 108 203 203 203 F F F
126282024-02-27T21:48:52.102Z 8 160-179 116 116 113 213 213 208 F F F <---
126292024-02-27T21:48:52.105Z 9 180-199 113 113 113 208 208 208 F F F
126302024-02-27T21:48:52.105Z 10 200-219 113 113 113 208 208 208 F F F
126312024-02-27T21:48:52.105Z 11 220-239 115 115 115 212 212 212 T T F <---
126322024-02-27T21:48:52.105Z 12 240-259 115 115 115 212 212 212 F F F
126332024-02-27T21:48:52.105Z 13 260-279 115 115 115 212 212 212 F F F
126342024-02-27T21:48:52.105Z 14 280-299 115 115 115 212 212 212 F F F
126352024-02-27T21:48:52.105Z 15 300-319 111 111 111 206 206 206 F F F
126362024-02-27T21:48:52.105Z 16 320-339 115 115 115 212 212 212 F F F
126372024-02-27T21:48:52.105Z 17 340-359 115 115 115 212 212 212 F F F
126382024-02-27T21:48:52.105Z 18 360-379 107 107 107 202 202 202 F F F
126392024-02-27T21:48:52.105Z 19 380-399 116 116 115 215 215 212 F F F <---
126402024-02-27T21:48:52.105Z 20 400-419 115 115 115 212 212 212 F F F
126412024-02-27T21:48:52.105Z 21 420-439 116 116 115 215 215 212 F F F <---
126422024-02-27T21:48:52.106Z 22 440-459 113 113 113 208 208 208 F F F
126432024-02-27T21:48:52.106Z 23 460-479 113 113 113 208 208 208 F F F
126442024-02-27T21:48:52.106Z 24 480-499 108 108 108 203 203 203 F F F
126452024-02-27T21:48:52.106Z 25 500-519 115 115 115 212 212 212 F F F
126462024-02-27T21:48:52.106Z 26 520-539 114 114 114 209 209 209 F F F
126472024-02-27T21:48:52.106Z 27 540-559 115 115 115 212 212 212 F F F
126482024-02-27T21:48:52.106Z 28 560-579 115 115 115 212 212 212 F F F
126492024-02-27T21:48:52.106Z 29 580-599 116 116 113 213 213 208 F F F <---
126502024-02-27T21:48:52.106ZMax gen: 116, Max flush: 215
126512024-02-27T21:48:52.106ZError: Difference in extent metadata found!
126522024-02-27T21:48:52.106ZA Difference in extent metadata is expected here
126532024-02-27T21:48:52.106ZOn loop 58
126542024-02-27T21:48:52.106Z
126552024-02-27T21:48:52.109ZVerifying data now
126562024-02-27T21:48:56.690ZNo change after verify
126572024-02-27T21:48:56.690ZLoop: 58 Downstairs dump after verify (and repair):
126582024-02-27T21:48:56.706ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126592024-02-27T21:48:56.706Z 0 000-019 116 116 116 213 213 213 F F F
126602024-02-27T21:48:56.706Z 1 020-039 116 116 116 213 213 213 F F F
126612024-02-27T21:48:56.706Z 2 040-059 117 117 117 216 216 216 F F F
126622024-02-27T21:48:56.706Z 3 060-079 117 117 117 216 216 216 F F F
126632024-02-27T21:48:56.706Z 4 080-099 116 116 116 213 213 213 F F F
126642024-02-27T21:48:56.706Z 5 100-119 116 116 116 213 213 213 F F F
126652024-02-27T21:48:56.706Z 6 120-139 115 115 115 212 212 212 F F F
126662024-02-27T21:48:56.706Z 7 140-159 108 108 108 203 203 203 F F F
126672024-02-27T21:48:56.713Z 8 160-179 116 116 116 213 213 213 F F F
126682024-02-27T21:48:56.713Z 9 180-199 113 113 113 208 208 208 F F F
126692024-02-27T21:48:56.713Z 10 200-219 113 113 113 208 208 208 F F F
126702024-02-27T21:48:56.713Z 11 220-239 117 117 117 216 216 216 F F F
126712024-02-27T21:48:56.713Z 12 240-259 115 115 115 212 212 212 F F F
126722024-02-27T21:48:56.713Z 13 260-279 115 115 115 212 212 212 F F F
126732024-02-27T21:48:56.713Z 14 280-299 115 115 115 212 212 212 F F F
126742024-02-27T21:48:56.713Z 15 300-319 111 111 111 206 206 206 F F F
126752024-02-27T21:48:56.713Z 16 320-339 115 115 115 212 212 212 F F F
126762024-02-27T21:48:56.713Z 17 340-359 115 115 115 212 212 212 F F F
126772024-02-27T21:48:56.713Z 18 360-379 107 107 107 202 202 202 F F F
126782024-02-27T21:48:56.713Z 19 380-399 116 116 116 215 215 215 F F F
126792024-02-27T21:48:56.713Z 20 400-419 115 115 115 212 212 212 F F F
126802024-02-27T21:48:56.713Z 21 420-439 116 116 116 215 215 215 F F F
126812024-02-27T21:48:56.713Z 22 440-459 113 113 113 208 208 208 F F F
126822024-02-27T21:48:56.713Z 23 460-479 113 113 113 208 208 208 F F F
126832024-02-27T21:48:56.713Z 24 480-499 108 108 108 203 203 203 F F F
126842024-02-27T21:48:56.713Z 25 500-519 115 115 115 212 212 212 F F F
126852024-02-27T21:48:56.713Z 26 520-539 114 114 114 209 209 209 F F F
126862024-02-27T21:48:56.713Z 27 540-559 115 115 115 212 212 212 F F F
126872024-02-27T21:48:56.714Z 28 560-579 115 115 115 212 212 212 F F F
126882024-02-27T21:48:56.714Z 29 580-599 116 116 116 213 213 213 F F F
126892024-02-27T21:48:56.714ZMax gen: 117, Max flush: 216
126902024-02-27T21:48:56.714Z
126912024-02-27T21:48:56.714ZBegin loop 59
126922024-02-27T21:48:56.714ZDownstairs to restart: 2
126932024-02-27T21:48:56.727Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105158975Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619}
126942024-02-27T21:48:56.727Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
126952024-02-27T21:48:56.728Z,"time":"2024-02-27T21:48:55.105458869Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619}
126962024-02-27T21:48:56.728Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105498735Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619}
126972024-02-27T21:48:56.736Z{"msg":"Crucible 8b552c2b-e0c8-4789-86ec-b069a7c36382 has session id: b1b85e90-9c50-43f3-949f-0bcf873e6193","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105526981Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
126982024-02-27T21:48:56.736Z{"msg":"Upstairs opts: Upstairs UUID: 8b552c2b-e0c8-4789-86ec-b069a7c36382, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105571756Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
126992024-02-27T21:48:56.737Z{"msg":"Crucible stats registered with UUID: 8b552c2b-e0c8-4789-86ec-b069a7c36382","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.10562167Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127002024-02-27T21:48:56.737Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105657546Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619}
127012024-02-27T21:48:56.737Z{"msg":"8b552c2b-e0c8-4789-86ec-b069a7c36382 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105732097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127022024-02-27T21:48:56.737Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105777302Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127032024-02-27T21:48:56.737Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105807588Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127042024-02-27T21:48:56.737Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105829456Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127052024-02-27T21:48:56.737Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105857392Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127062024-02-27T21:48:56.737Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105872561Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127072024-02-27T21:48:56.737Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.105945772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127082024-02-27T21:48:56.737Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.106041001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127092024-02-27T21:48:56.737Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.106111792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127102024-02-27T21:48:56.737Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.106142909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127112024-02-27T21:48:56.737Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.106169066Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127122024-02-27T21:48:56.738Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.106194833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127132024-02-27T21:48:56.738Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.10630475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127142024-02-27T21:48:56.738Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.106796882Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127152024-02-27T21:48:56.738Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.106833237Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127162024-02-27T21:48:56.738Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.106854125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127172024-02-27T21:48:56.738Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.106889851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127182024-02-27T21:48:56.738Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.107074379Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127192024-02-27T21:48:56.738Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.107110535Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127202024-02-27T21:48:56.738Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.107498259Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127212024-02-27T21:48:56.738Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.107548163Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127222024-02-27T21:48:56.738Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.107813672Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127232024-02-27T21:48:56.738Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108115146Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127242024-02-27T21:48:56.738Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.10816722Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127252024-02-27T21:48:56.739Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108230103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127262024-02-27T21:48:56.739Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108273608Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127272024-02-27T21:48:56.739Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108319312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127282024-02-27T21:48:56.739Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108357328Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127292024-02-27T21:48:56.739Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108378945Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127302024-02-27T21:48:56.739Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108412731Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127312024-02-27T21:48:56.739Z{"msg":"[0]R flush_numbers[0..12]: [213, 213, 216, 216, 213, 213, 212, 203, 213, 208, 208, 216]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108449217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127322024-02-27T21:48:56.739Z{"msg":"[0]R generation[0..12]: [116, 116, 117, 117, 116, 116, 115, 108, 116, 113, 113, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108484903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127332024-02-27T21:48:56.739Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108521589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127342024-02-27T21:48:56.739ZWait for a query_work_queue command to finish before sending IO
127352024-02-27T21:48:56.739Z{"msg":"[1]R flush_numbers[0..12]: [213, 213, 216, 216, 213, 213, 212, 203, 213, 208, 208, 216]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108563484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127362024-02-27T21:48:56.739Z{"msg":"[1]R generation[0..12]: [116, 116, 117, 117, 116, 116, 115, 108, 116, 113, 113, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108604769Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127372024-02-27T21:48:56.739Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108641444Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127382024-02-27T21:48:56.740Z{"msg":"[2]R flush_numbers[0..12]: [213, 213, 216, 216, 213, 213, 212, 203, 213, 208, 208, 216]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.1086792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127392024-02-27T21:48:56.740Z{"msg":"[2]R generation[0..12]: [116, 116, 117, 117, 116, 116, 115, 108, 116, 113, 113, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108710076Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127402024-02-27T21:48:56.740Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108725934Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127412024-02-27T21:48:56.740Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108740163Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127422024-02-27T21:48:56.740Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108754431Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127432024-02-27T21:48:56.740Z{"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108767489Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127442024-02-27T21:48:56.740Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108788727Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"":"downstairs","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127452024-02-27T21:48:56.740Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108802665Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127462024-02-27T21:48:56.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108815614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"0","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127472024-02-27T21:48:56.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108829622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"1","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127482024-02-27T21:48:56.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.10885192Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"client":"2","":"downstairs","session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127492024-02-27T21:48:56.740Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108880766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127502024-02-27T21:48:56.740Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108894814Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127512024-02-27T21:48:56.741Z{"msg":"8b552c2b-e0c8-4789-86ec-b069a7c36382 is now active with session: b1b85e90-9c50-43f3-949f-0bcf873e6193","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108908373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"b1b85e90-9c50-43f3-949f-0bcf873e6193"}
127522024-02-27T21:48:56.741Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108927061Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127532024-02-27T21:48:56.741Zb1b85e90-9c50-43f3-949f-0bcf873e6193"}
127542024-02-27T21:48:56.741Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:48:55.108946708Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619}
127552024-02-27T21:48:56.741ZLoading write count information from file "/tmp/test_repair_verify.data"
127562024-02-27T21:48:56.741ZRun Repair workload
127572024-02-27T21:48:56.741Z01/30 Write block 477 len 8192 data: 7 8
127582024-02-27T21:48:56.745Z02/30 Write block 520 len 40960 data: 9 6 8 6 7 6 6 6 7 5
127592024-02-27T21:48:56.745Z03/30 Read block 556 len 36864
127602024-02-27T21:48:56.745Z04/30 Write block 501 len 24576 data: 4 5 5 7 8 9
127612024-02-27T21:48:56.745Z05/30 Read block 152 len 36864
127622024-02-27T21:48:56.745Z06/30 Flush
127632024-02-27T21:48:56.746Z07/30 Write block 222 len 16384 data: 11 10 10 7
127642024-02-27T21:48:56.746Z08/30 Write block 577 len 8192 data: 7 9
127652024-02-27T21:48:56.746Z09/30 Write block 191 len 12288 data: 11 10 8
127662024-02-27T21:48:56.746Z10/30 Read block 298 len 4096
127672024-02-27T21:48:56.746Z11/30 Write block 320 len 24576 data: 14 12 10 9 8 7
127682024-02-27T21:48:56.746Z12/30 Write block 225 len 16384 data: 8 9 9 10
127692024-02-27T21:48:56.746Z13/30 Read block 412 len 28672
127702024-02-27T21:48:56.746Z14/30 Flush
127712024-02-27T21:48:56.750Z15/30 Write block 188 len 4096 data: 13
127722024-02-27T21:48:56.750Z16/30 Read block 465 len 12288
127732024-02-27T21:48:56.750Z17/30 Read block 506 len 24576
127742024-02-27T21:48:56.754Z18/30 Write block 250 len 40960 data: 11 10 10 10 8 12 12 9 12 11
127752024-02-27T21:48:56.754Z19/30 Write block 340 len 40960 data: 14 14 11 10 10 8 8 9 12 13
127762024-02-27T21:48:56.754Z20/30 Write block 293 len 24576 data: 10 9 10 10 10 10
127772024-02-27T21:48:56.754Z21/30 Read block 349 len 20480
127782024-02-27T21:48:56.757Z22/30 Read block 555 len 12288
127792024-02-27T21:48:56.757Z23/30 Write block 327 len 40960 data: 7 9 9 9 9 9 11 11 14 11
127802024-02-27T21:48:56.757Z24/30 Read block 587 len 32768
127812024-02-27T21:48:56.761Z25/30 Read block 538 len 40960
127822024-02-27T21:48:56.761Z26/30 Read block 500 len 40960
127832024-02-27T21:48:56.761Z27/30 Read block 489 len 16384
127842024-02-27T21:48:56.761Z28/30 Read block 71 len 20480
127852024-02-27T21:48:56.766Z29/30 Read block 553 len 40960
127862024-02-27T21:48:56.766Z30/30 Write block 51 len 4096 data: 10
127872024-02-27T21:48:56.766Z----------------------------------------------------------------
127882024-02-27T21:48:56.766Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:24
127892024-02-27T21:48:56.766ZStates: Active Active Active
127902024-02-27T21:48:56.766ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127912024-02-27T21:48:56.766Z 7 Acked 1006 Write 4 Done Done Done false
127922024-02-27T21:48:56.766Z 8 Acked 1007 Write 2 Done Done Done false
127932024-02-27T21:48:56.766Z 9 Acked 1008 Write 3 Done Done Done false
127942024-02-27T21:48:56.766Z 10 Acked 1009 Read 1 Done Done Done false
127952024-02-27T21:48:56.766Z 11 Acked 1010 Write 6 Done Done Sent false
127962024-02-27T21:48:56.766Z 12 Acked 1011 Write 4 Done Done Sent false
127972024-02-27T21:48:56.766Z 13 Acked 1012 Read 7 Done Done Sent false
127982024-02-27T21:48:56.766Z 14 Acked 1013 Flush 0 Done Done Sent false
127992024-02-27T21:48:56.766Z 15 Acked 1014 Write 1 Done Done Sent false
128002024-02-27T21:48:56.766Z 16 Acked 1015 Read 3 Done Done Sent false
128012024-02-27T21:48:56.766Z 17 Acked 1016 Read 6 Done Done Sent false
128022024-02-27T21:48:56.766Z 18 Acked 1017 Write 10 Done Done Sent false
128032024-02-27T21:48:56.766Z 19 Acked 1018 Write 10 Done Done Sent false
128042024-02-27T21:48:56.767Z 20 Acked 1019 Write 6 Done Done Sent false
128052024-02-27T21:48:56.767Z 21 Acked 1020 Read 5 Done Done Sent false
128062024-02-27T21:48:56.767Z 22 Acked 1021 Read 3 Done Done Sent false
128072024-02-27T21:48:56.767Z 23 Acked 1022 Write 10 Done Done Sent false
128082024-02-27T21:48:56.767Z 24 Acked 1023 Read 8 Done Done Sent false
128092024-02-27T21:48:56.767Z 25 Acked 1024 Read 10 Done Done Sent false
128102024-02-27T21:48:56.767Z 26 Acked 1025 Read 10 Done Done Sent false
128112024-02-27T21:48:56.767Z 27 Acked 1026 Read 4 Done Done Sent false
128122024-02-27T21:48:56.767Z 28 Acked 1027 Read 5 Done Done Sent false
128132024-02-27T21:48:56.767Z 29 Acked 1028 Read 10 Done Done Sent false
128142024-02-27T21:48:56.767Z 30 Acked 1029 Write 1 Sent Sent Sent false
128152024-02-27T21:48:56.767Z STATES DS:0 DS:1 DS:2 TOTAL
128162024-02-27T21:48:56.767Z New 0 0 0 0
128172024-02-27T21:48:56.767Z Sent 1 1 20 22
128182024-02-27T21:48:56.767Z Done 23 23 4 50
128192024-02-27T21:48:56.767Z Skipped 0 0 0 0
128202024-02-27T21:48:56.767Z Error 0 0 0 0
128212024-02-27T21:48:56.768ZLast Flush: 1013 1013 1005
128222024-02-27T21:48:56.768ZDownstairs last five completed: 1005 1004 1003 1002 1001
128232024-02-27T21:48:56.768ZUpstairs last five completed: 30 29 28 27 26
128242024-02-27T21:48:56.768Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:48:55.141327624Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1619}
128252024-02-27T21:48:56.768ZWrote out file "/tmp/test_repair_verify.data"
128262024-02-27T21:48:56.771Z
128272024-02-27T21:48:56.777ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128282024-02-27T21:48:56.789ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128292024-02-27T21:48:56.789Z 0 000-019 116 116 116 213 213 213 F F F
128302024-02-27T21:48:56.789Z 1 020-039 116 116 116 213 213 213 F F F
128312024-02-27T21:48:56.789Z 2 040-059 117 117 117 216 216 216 T T F <---
128322024-02-27T21:48:56.789Z 3 060-079 117 117 117 216 216 216 F F F
128332024-02-27T21:48:56.789Z 4 080-099 116 116 116 213 213 213 F F F
128342024-02-27T21:48:56.789Z 5 100-119 116 116 116 213 213 213 F F F
128352024-02-27T21:48:56.789Z 6 120-139 115 115 115 212 212 212 F F F
128362024-02-27T21:48:56.789Z 7 140-159 108 108 108 203 203 203 F F F
128372024-02-27T21:48:56.789Z 8 160-179 116 116 116 213 213 213 F F F
128382024-02-27T21:48:56.793Z 9 180-199 118 118 113 218 218 208 T T T <---
128392024-02-27T21:48:56.793Z 10 200-219 113 113 113 208 208 208 F F F
128402024-02-27T21:48:56.793Z 11 220-239 118 118 117 218 218 216 F F T <---
128412024-02-27T21:48:56.793Z 12 240-259 115 115 115 212 212 212 T T F <---
128422024-02-27T21:48:56.793Z 13 260-279 115 115 115 212 212 212 F F F
128432024-02-27T21:48:56.793Z 14 280-299 115 115 115 212 212 212 T T F <---
128442024-02-27T21:48:56.793Z 15 300-319 111 111 111 206 206 206 F F F
128452024-02-27T21:48:56.794Z 16 320-339 118 118 115 218 218 212 T T F <---
128462024-02-27T21:48:56.794Z 17 340-359 115 115 115 212 212 212 T T F <---
128472024-02-27T21:48:56.794Z 18 360-379 107 107 107 202 202 202 F F F
128482024-02-27T21:48:56.794Z 19 380-399 116 116 116 215 215 215 F F F
128492024-02-27T21:48:56.794Z 20 400-419 115 115 115 212 212 212 F F F
128502024-02-27T21:48:56.794Z 21 420-439 116 116 116 215 215 215 F F F
128512024-02-27T21:48:56.794Z 22 440-459 113 113 113 208 208 208 F F F
128522024-02-27T21:48:56.794Z 23 460-479 118 118 118 217 217 217 F F F
128532024-02-27T21:48:56.794Z 24 480-499 108 108 108 203 203 203 F F F
128542024-02-27T21:48:56.794Z 25 500-519 118 118 118 217 217 217 F F F
128552024-02-27T21:48:56.794Z 26 520-539 118 118 118 217 217 217 F F F
128562024-02-27T21:48:56.794Z 27 540-559 115 115 115 212 212 212 F F F
128572024-02-27T21:48:56.794Z 28 560-579 118 118 115 218 218 212 F F T <---
128582024-02-27T21:48:56.795Z 29 580-599 116 116 116 213 213 213 F F F
128592024-02-27T21:48:56.795ZMax gen: 118, Max flush: 218
128602024-02-27T21:48:56.795ZError: Difference in extent metadata found!
128612024-02-27T21:48:56.795ZA Difference in extent metadata is expected here
128622024-02-27T21:48:56.795ZOn loop 59
128632024-02-27T21:48:56.795Z
128642024-02-27T21:48:56.799ZVerifying data now
128652024-02-27T21:49:02.249ZNo change after verify
128662024-02-27T21:49:02.250ZLoop: 59 Downstairs dump after verify (and repair):
128672024-02-27T21:49:02.264ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128682024-02-27T21:49:02.264Z 0 000-019 116 116 116 213 213 213 F F F
128692024-02-27T21:49:02.264Z 1 020-039 116 116 116 213 213 213 F F F
128702024-02-27T21:49:02.264Z 2 040-059 119 119 119 219 219 219 F F F
128712024-02-27T21:49:02.264Z 3 060-079 117 117 117 216 216 216 F F F
128722024-02-27T21:49:02.264Z 4 080-099 116 116 116 213 213 213 F F F
128732024-02-27T21:49:02.264Z 5 100-119 116 116 116 213 213 213 F F F
128742024-02-27T21:49:02.264Z 6 120-139 115 115 115 212 212 212 F F F
128752024-02-27T21:49:02.264Z 7 140-159 108 108 108 203 203 203 F F F
128762024-02-27T21:49:02.268Z 8 160-179 116 116 116 213 213 213 F F F
128772024-02-27T21:49:02.268Z 9 180-199 119 119 119 219 219 219 F F F
128782024-02-27T21:49:02.268Z 10 200-219 113 113 113 208 208 208 F F F
128792024-02-27T21:49:02.268Z 11 220-239 118 118 118 218 218 218 F F F
128802024-02-27T21:49:02.268Z 12 240-259 119 119 119 219 219 219 F F F
128812024-02-27T21:49:02.268Z 13 260-279 115 115 115 212 212 212 F F F
128822024-02-27T21:49:02.268Z 14 280-299 119 119 119 219 219 219 F F F
128832024-02-27T21:49:02.268Z 15 300-319 111 111 111 206 206 206 F F F
128842024-02-27T21:49:02.268Z 16 320-339 119 119 119 219 219 219 F F F
128852024-02-27T21:49:02.268Z 17 340-359 119 119 119 219 219 219 F F F
128862024-02-27T21:49:02.268Z 18 360-379 107 107 107 202 202 202 F F F
128872024-02-27T21:49:02.268Z 19 380-399 116 116 116 215 215 215 F F F
128882024-02-27T21:49:02.268Z 20 400-419 115 115 115 212 212 212 F F F
128892024-02-27T21:49:02.268Z 21 420-439 116 116 116 215 215 215 F F F
128902024-02-27T21:49:02.268Z 22 440-459 113 113 113 208 208 208 F F F
128912024-02-27T21:49:02.269Z 23 460-479 118 118 118 217 217 217 F F F
128922024-02-27T21:49:02.269Z 24 480-499 108 108 108 203 203 203 F F F
128932024-02-27T21:49:02.269Z 25 500-519 118 118 118 217 217 217 F F F
128942024-02-27T21:49:02.269Z 26 520-539 118 118 118 217 217 217 F F F
128952024-02-27T21:49:02.269Z 27 540-559 115 115 115 212 212 212 F F F
128962024-02-27T21:49:02.269Z 28 560-579 118 118 118 218 218 218 F F F
128972024-02-27T21:49:02.269Z 29 580-599 116 116 116 213 213 213 F F F
128982024-02-27T21:49:02.269ZMax gen: 119, Max flush: 219
128992024-02-27T21:49:02.269Z
129002024-02-27T21:49:02.269ZBegin loop 60
129012024-02-27T21:49:02.269ZDownstairs to restart: 0
129022024-02-27T21:49:02.287Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663132088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627}
129032024-02-27T21:49:02.287Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663431183Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627}
129042024-02-27T21:49:02.287ZCrucible runtime is spawned
129052024-02-27T21:49:02.298Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663499315Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627}
129062024-02-27T21:49:02.298Z{"msg":"Crucible a7dac212-05eb-4118-b7dd-ff78b54d54e6 has session id: c90b9316-81cd-4cd7-842f-c1f5c6f52f68","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663545239Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129072024-02-27T21:49:02.298Z{"msg":"Upstairs opts: Upstairs UUID: a7dac212-05eb-4118-b7dd-ff78b54d54e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663611541Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129082024-02-27T21:49:02.298Z{"msg":"Crucible stats registered with UUID: a7dac212-05eb-4118-b7dd-ff78b54d54e6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663643207Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129092024-02-27T21:49:02.298Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663672364Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627}
129102024-02-27T21:49:02.298Z{"msg":"a7dac212-05eb-4118-b7dd-ff78b54d54e6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663719259Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129112024-02-27T21:49:02.298Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663764393Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129122024-02-27T21:49:02.298Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663814258Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129132024-02-27T21:49:02.298Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663858042Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129142024-02-27T21:49:02.298Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663892508Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129152024-02-27T21:49:02.298Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663938833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129162024-02-27T21:49:02.298Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.663985237Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129172024-02-27T21:49:02.298Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.664071607Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129182024-02-27T21:49:02.298Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.66412967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129192024-02-27T21:49:02.298Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.664177335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129202024-02-27T21:49:02.298Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.664210571Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129212024-02-27T21:49:02.298Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.664249186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129222024-02-27T21:49:02.298Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.664310289Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129232024-02-27T21:49:02.298Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.664910918Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129242024-02-27T21:49:02.298Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.664959423Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129252024-02-27T21:49:02.298Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.664990419Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129262024-02-27T21:49:02.299Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.665023485Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129272024-02-27T21:49:02.299Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.665039313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129282024-02-27T21:49:02.299Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.665059721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129292024-02-27T21:49:02.299Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.665483751Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129302024-02-27T21:49:02.299Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.665531835Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129312024-02-27T21:49:02.299Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.665799424Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129322024-02-27T21:49:02.299Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666010499Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129332024-02-27T21:49:02.299Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666050294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129342024-02-27T21:49:02.299Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666078571Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129352024-02-27T21:49:02.299Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666109187Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129362024-02-27T21:49:02.299Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666134214Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129372024-02-27T21:49:02.299Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.66616733Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129382024-02-27T21:49:02.299Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666247281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129392024-02-27T21:49:02.299Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666293286Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129402024-02-27T21:49:02.299Z{"msg":"[0]R flush_numbers[0..12]: [213, 213, 219, 216, 213, 213, 212, 203, 213, 219, 208, 218]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666312103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129412024-02-27T21:49:02.299Z{"msg":"[0]R generation[0..12]: [116, 116, 119, 117, 116, 116, 115, 108, 116, 119, 113, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666334921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129422024-02-27T21:49:02.299Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666349939Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129432024-02-27T21:49:02.299Z{"msg":"[1]R flush_numbers[0..12]: [213, 213, 219, 216, 213, 213, 212, 203, 213, 219, 208, 218]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666364927Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129442024-02-27T21:49:02.299Z{"msg":"[1]R generation[0..12]: [116, 116, 119, 117, 116, 116, 115, 108, 116, 119, 113, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666378975Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129452024-02-27T21:49:02.299ZWait for a query_work_queue command to finish before sending IO
129462024-02-27T21:49:02.299Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666402313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129472024-02-27T21:49:02.299Z{"msg":"[2]R flush_numbers[0..12]: [213, 213, 219, 216, 213, 213, 212, 203, 213, 219, 208, 218]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666417101Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129482024-02-27T21:49:02.299Z{"msg":"[2]R generation[0..12]: [116, 116, 119, 117, 116, 116, 115, 108, 116, 119, 113, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666437249Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129492024-02-27T21:49:02.299Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666452987Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129502024-02-27T21:49:02.299Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666486263Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129512024-02-27T21:49:02.300Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.66651252Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129522024-02-27T21:49:02.300Z{"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666535387Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129532024-02-27T21:49:02.300Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666559034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"":"downstairs","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129542024-02-27T21:49:02.300Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.66659314Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129552024-02-27T21:49:02.300Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666618257Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"0","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129562024-02-27T21:49:02.300Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666650523Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"1","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129572024-02-27T21:49:02.300Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666686939Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"client":"2","":"downstairs","session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129582024-02-27T21:49:02.300Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666722515Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129592024-02-27T21:49:02.300Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666756131Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129602024-02-27T21:49:02.300Z
129612024-02-27T21:49:02.300Z{"msg":"a7dac212-05eb-4118-b7dd-ff78b54d54e6 is now active with session: c90b9316-81cd-4cd7-842f-c1f5c6f52f68","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666790417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129622024-02-27T21:49:02.300Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666802695Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627,"session_id":"c90b9316-81cd-4cd7-842f-c1f5c6f52f68"}
129632024-02-27T21:49:02.300Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:00.666822053Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627}
129642024-02-27T21:49:02.300ZLoading write count information from file "/tmp/test_repair_verify.data"
129652024-02-27T21:49:02.300ZRun Repair workload
129662024-02-27T21:49:02.300Z01/30 Write block 284 len 4096 data: 6
129672024-02-27T21:49:02.307Z02/30 Write block 331 len 20480 data: 10 10 12 12 15
129682024-02-27T21:49:02.307Z03/30 Read block 128 len 4096
129692024-02-27T21:49:02.307Z04/30 Read block 431 len 40960
129702024-02-27T21:49:02.307Z05/30 Write block 526 len 40960 data: 7 7 8 6 5 5 4 5 8 9
129712024-02-27T21:49:02.307Z06/30 Write block 519 len 12288 data: 7 10 7
129722024-02-27T21:49:02.307Z07/30 Write block 106 len 32768 data: 11 14 13 12 11 11 9 8
129732024-02-27T21:49:02.307Z08/30 Read block 14 len 24576
129742024-02-27T21:49:02.307Z09/30 Write block 394 len 8192 data: 9 9
129752024-02-27T21:49:02.307Z10/30 Read block 319 len 4096
129762024-02-27T21:49:02.307Z11/30 Read block 308 len 16384
129772024-02-27T21:49:02.307Z12/30 Read block 114 len 28672
129782024-02-27T21:49:02.307Z13/30 Read block 32 len 16384
129792024-02-27T21:49:02.307Z14/30 Read block 131 len 24576
129802024-02-27T21:49:02.307Z15/30 Read block 357 len 8192
129812024-02-27T21:49:02.308Z16/30 Write block 471 len 4096 data: 7
129822024-02-27T21:49:02.308Z17/30 Write block 0 len 20480 data: 2 3 5 5 7
129832024-02-27T21:49:02.308Z18/30 Read block 580 len 20480
129842024-02-27T21:49:02.308Z19/30 Write block 92 len 4096 data: 6
129852024-02-27T21:49:02.314Z20/30 Read block 83 len 32768
129862024-02-27T21:49:02.315Z21/30 Flush
129872024-02-27T21:49:02.315Z22/30 Write block 178 len 8192 data: 14 12
129882024-02-27T21:49:02.315Z23/30 Write block 58 len 20480 data: 11 9 7 5 6
129892024-02-27T21:49:02.315Z24/30 Flush
129902024-02-27T21:49:02.315Z25/30 Write block 336 len 16384 data: 12 12 13 12
129912024-02-27T21:49:02.315Z26/30 Read block 101 len 24576
129922024-02-27T21:49:02.319Z27/30 Read block 555 len 8192
129932024-02-27T21:49:02.319Z28/30 Write block 467 len 40960 data: 9 9 10 7 8 6 7 6 8 9
129942024-02-27T21:49:02.319Z29/30 Read block 248 len 28672
129952024-02-27T21:49:02.319Z30/30 Read block 367 len 4096
129962024-02-27T21:49:02.323Z----------------------------------------------------------------
129972024-02-27T21:49:02.324Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
129982024-02-27T21:49:02.324ZStates: Active Active Active
129992024-02-27T21:49:02.324ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
130002024-02-27T21:49:02.324Z 1 Acked 1000 Write 1 Sent Done Done false
130012024-02-27T21:49:02.324Z 2 Acked 1001 Write 5 Sent Done Done false
130022024-02-27T21:49:02.324Z 3 Acked 1002 Read 1 Sent Done Done false
130032024-02-27T21:49:02.324Z 4 Acked 1003 Read 10 Sent Done Done false
130042024-02-27T21:49:02.324Z 5 Acked 1004 Write 10 Sent Done Done false
130052024-02-27T21:49:02.324Z 6 Acked 1005 Write 3 Sent Done Done false
130062024-02-27T21:49:02.324Z 7 Acked 1006 Write 8 Sent Done Done false
130072024-02-27T21:49:02.324Z 8 Acked 1007 Read 6 Sent Done Done false
130082024-02-27T21:49:02.324Z 9 Acked 1008 Write 2 Sent Done Done false
130092024-02-27T21:49:02.324Z 10 Acked 1009 Read 1 Sent Done Done false
130102024-02-27T21:49:02.324Z 11 Acked 1010 Read 4 Sent Done Done false
130112024-02-27T21:49:02.324Z 12 Acked 1011 Read 7 Sent Done Done false
130122024-02-27T21:49:02.324Z 13 Acked 1012 Read 4 Sent Done Done false
130132024-02-27T21:49:02.324Z 14 Acked 1013 Read 6 Sent Done Done false
130142024-02-27T21:49:02.324Z 15 Acked 1014 Read 2 Sent Done Done false
130152024-02-27T21:49:02.324Z 16 Acked 1015 Write 1 Sent Done Done false
130162024-02-27T21:49:02.324Z 17 Acked 1016 Write 5 Sent Done Done false
130172024-02-27T21:49:02.324Z 18 Acked 1017 Read 5 Sent Done Done false
130182024-02-27T21:49:02.324Z 19 Acked 1018 Write 1 Sent Done Done false
130192024-02-27T21:49:02.324Z 20 Acked 1019 Read 8 Sent Done Done false
130202024-02-27T21:49:02.324Z 21 Acked 1020 Flush 0 Sent Done Done false
130212024-02-27T21:49:02.324Z 22 Acked 1021 Write 2 Sent Done Done false
130222024-02-27T21:49:02.325Z 23 Acked 1022 Write 5 Sent Done Done false
130232024-02-27T21:49:02.325Z 24 Acked 1023 Flush 0 Sent Done Done false
130242024-02-27T21:49:02.325Z 25 Acked 1024 Write 4 Sent Done Done false
130252024-02-27T21:49:02.325Z 26 Acked 1025 Read 6 Sent Done Done false
130262024-02-27T21:49:02.325Z 27 Acked 1026 Read 2 Sent Done Done false
130272024-02-27T21:49:02.325Z 28 Acked 1027 Write 10 Sent Done Done false
130282024-02-27T21:49:02.325Z 29 Acked 1028 Read 7 Sent Done Done false
130292024-02-27T21:49:02.325Z 30 Acked 1029 Read 1 Sent Done Done false
130302024-02-27T21:49:02.325Z STATES DS:0 DS:1 DS:2 TOTAL
130312024-02-27T21:49:02.325Z New 0 0 0 0
130322024-02-27T21:49:02.325Z Sent 30 0 0 30
130332024-02-27T21:49:02.325Z Done 0 30 30 60
130342024-02-27T21:49:02.325Z Skipped 0 0 0 0
130352024-02-27T21:49:02.325Z Error 0 0 0 0
130362024-02-27T21:49:02.325ZLast Flush: 0 1023 1023
130372024-02-27T21:49:02.325ZDownstairs last five completed:
130382024-02-27T21:49:02.325ZUpstairs last five completed: 30 29 28 27 26
130392024-02-27T21:49:02.325Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:00.696158168Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1627}
130402024-02-27T21:49:02.325ZWrote out file "/tmp/test_repair_verify.data"
130412024-02-27T21:49:02.331Z
130422024-02-27T21:49:02.331ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130432024-02-27T21:49:02.354ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130442024-02-27T21:49:02.355Z 0 000-019 116 120 120 213 220 220 F F F <---
130452024-02-27T21:49:02.355Z 1 020-039 116 116 116 213 213 213 F F F
130462024-02-27T21:49:02.355Z 2 040-059 119 120 120 219 221 221 F F F <---
130472024-02-27T21:49:02.355Z 3 060-079 117 120 120 216 221 221 F F F <---
130482024-02-27T21:49:02.355Z 4 080-099 116 120 120 213 220 220 F F F <---
130492024-02-27T21:49:02.355Z 5 100-119 116 120 120 213 220 220 F F F <---
130502024-02-27T21:49:02.355Z 6 120-139 115 115 115 212 212 212 F F F
130512024-02-27T21:49:02.355Z 7 140-159 108 108 108 203 203 203 F F F
130522024-02-27T21:49:02.355Z 8 160-179 116 120 120 213 221 221 F F F <---
130532024-02-27T21:49:02.355Z 9 180-199 119 119 119 219 219 219 F F F
130542024-02-27T21:49:02.371Z 10 200-219 113 113 113 208 208 208 F F F
130552024-02-27T21:49:02.372Z 11 220-239 118 118 118 218 218 218 F F F
130562024-02-27T21:49:02.372Z 12 240-259 119 119 119 219 219 219 F F F
130572024-02-27T21:49:02.372Z 13 260-279 115 115 115 212 212 212 F F F
130582024-02-27T21:49:02.372Z 14 280-299 119 120 120 219 220 220 F F F <---
130592024-02-27T21:49:02.372Z 15 300-319 111 111 111 206 206 206 F F F
130602024-02-27T21:49:02.372Z 16 320-339 119 120 120 219 220 220 F T T <---
130612024-02-27T21:49:02.372Z 17 340-359 119 119 119 219 219 219 F F F
130622024-02-27T21:49:02.372Z 18 360-379 107 107 107 202 202 202 F F F
130632024-02-27T21:49:02.372Z 19 380-399 116 120 120 215 220 220 F F F <---
130642024-02-27T21:49:02.372Z 20 400-419 115 115 115 212 212 212 F F F
130652024-02-27T21:49:02.372Z 21 420-439 116 116 116 215 215 215 F F F
130662024-02-27T21:49:02.372Z 22 440-459 113 113 113 208 208 208 F F F
130672024-02-27T21:49:02.372Z 23 460-479 118 120 120 217 220 220 F T T <---
130682024-02-27T21:49:02.372Z 24 480-499 108 108 108 203 203 203 F F F
130692024-02-27T21:49:02.372Z 25 500-519 118 120 120 217 220 220 F F F <---
130702024-02-27T21:49:02.372Z 26 520-539 118 120 120 217 220 220 F F F <---
130712024-02-27T21:49:02.373Z 27 540-559 115 115 115 212 212 212 F F F
130722024-02-27T21:49:02.373Z 28 560-579 118 118 118 218 218 218 F F F
130732024-02-27T21:49:02.373Z 29 580-599 116 116 116 213 213 213 F F F
130742024-02-27T21:49:02.373ZMax gen: 120, Max flush: 221
130752024-02-27T21:49:02.373ZError: Difference in extent metadata found!
130762024-02-27T21:49:02.373ZA Difference in extent metadata is expected here
130772024-02-27T21:49:02.373ZOn loop 60
130782024-02-27T21:49:02.373Z
130792024-02-27T21:49:02.373ZVerifying data now
130802024-02-27T21:49:06.960ZNo change after verify
130812024-02-27T21:49:06.960ZLoop: 60 Downstairs dump after verify (and repair):
130822024-02-27T21:49:06.971ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130832024-02-27T21:49:06.971Z 0 000-019 120 120 120 220 220 220 F F F
130842024-02-27T21:49:06.971Z 1 020-039 116 116 116 213 213 213 F F F
130852024-02-27T21:49:06.971Z 2 040-059 120 120 120 221 221 221 F F F
130862024-02-27T21:49:06.975Z 3 060-079 120 120 120 221 221 221 F F F
130872024-02-27T21:49:06.975Z 4 080-099 120 120 120 220 220 220 F F F
130882024-02-27T21:49:06.975Z 5 100-119 120 120 120 220 220 220 F F F
130892024-02-27T21:49:06.975Z 6 120-139 115 115 115 212 212 212 F F F
130902024-02-27T21:49:06.975Z 7 140-159 108 108 108 203 203 203 F F F
130912024-02-27T21:49:06.975Z 8 160-179 120 120 120 221 221 221 F F F
130922024-02-27T21:49:06.975Z 9 180-199 119 119 119 219 219 219 F F F
130932024-02-27T21:49:06.975Z 10 200-219 113 113 113 208 208 208 F F F
130942024-02-27T21:49:06.975Z 11 220-239 118 118 118 218 218 218 F F F
130952024-02-27T21:49:06.975Z 12 240-259 119 119 119 219 219 219 F F F
130962024-02-27T21:49:06.975Z 13 260-279 115 115 115 212 212 212 F F F
130972024-02-27T21:49:06.975Z 14 280-299 120 120 120 220 220 220 F F F
130982024-02-27T21:49:06.975Z 15 300-319 111 111 111 206 206 206 F F F
130992024-02-27T21:49:06.975Z 16 320-339 121 121 121 222 222 222 F F F
131002024-02-27T21:49:06.975Z 17 340-359 119 119 119 219 219 219 F F F
131012024-02-27T21:49:06.975Z 18 360-379 107 107 107 202 202 202 F F F
131022024-02-27T21:49:06.975Z 19 380-399 120 120 120 220 220 220 F F F
131032024-02-27T21:49:06.975Z 20 400-419 115 115 115 212 212 212 F F F
131042024-02-27T21:49:06.975Z 21 420-439 116 116 116 215 215 215 F F F
131052024-02-27T21:49:06.976Z 22 440-459 113 113 113 208 208 208 F F F
131062024-02-27T21:49:06.976Z 23 460-479 121 121 121 222 222 222 F F F
131072024-02-27T21:49:06.976Z 24 480-499 108 108 108 203 203 203 F F F
131082024-02-27T21:49:06.976Z 25 500-519 120 120 120 220 220 220 F F F
131092024-02-27T21:49:06.976Z 26 520-539 120 120 120 220 220 220 F F F
131102024-02-27T21:49:06.976Z 27 540-559 115 115 115 212 212 212 F F F
131112024-02-27T21:49:06.976Z 28 560-579 118 118 118 218 218 218 F F F
131122024-02-27T21:49:06.976Z 29 580-599 116 116 116 213 213 213 F F F
131132024-02-27T21:49:06.976ZMax gen: 121, Max flush: 222
131142024-02-27T21:49:06.976Z
131152024-02-27T21:49:06.976ZBegin loop 61
131162024-02-27T21:49:06.976ZDownstairs to restart: 2
131172024-02-27T21:49:06.994Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.370381921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635}
131182024-02-27T21:49:06.994Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.370633891Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635}
131192024-02-27T21:49:07.001Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.370671827Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635}
131202024-02-27T21:49:07.001Z{"msg":"Crucible c3e280fe-4dc9-4542-8cc9-c38d288a7d1d has session id: 2916170c-255d-4ede-a503-7f620b8ce823","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
131212024-02-27T21:49:07.001Z2024-02-27T21:49:05.370704313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131222024-02-27T21:49:07.001Z{"msg":"Upstairs opts: Upstairs UUID: c3e280fe-4dc9-4542-8cc9-c38d288a7d1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.370758626Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131232024-02-27T21:49:07.002Z{"msg":"Crucible stats registered with UUID: c3e280fe-4dc9-4542-8cc9-c38d288a7d1d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.370798332Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131242024-02-27T21:49:07.002Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.37081511Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635}
131252024-02-27T21:49:07.002Z{"msg":"c3e280fe-4dc9-4542-8cc9-c38d288a7d1d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.370953783Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131262024-02-27T21:49:07.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371014946Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131272024-02-27T21:49:07.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371060961Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131282024-02-27T21:49:07.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371110615Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131292024-02-27T21:49:07.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371139312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131302024-02-27T21:49:07.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371175857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131312024-02-27T21:49:07.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371201814Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131322024-02-27T21:49:07.002Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.37123914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131332024-02-27T21:49:07.002Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371283604Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131342024-02-27T21:49:07.002Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.37132019Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131352024-02-27T21:49:07.002Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371359546Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131362024-02-27T21:49:07.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371388172Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131372024-02-27T21:49:07.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371458144Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131382024-02-27T21:49:07.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.371960325Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131392024-02-27T21:49:07.003Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.372010919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131402024-02-27T21:49:07.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.372046574Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131412024-02-27T21:49:07.003Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.372073262Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131422024-02-27T21:49:07.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.372374846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131432024-02-27T21:49:07.003Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.372412482Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131442024-02-27T21:49:07.003Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.372577752Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131452024-02-27T21:49:07.003Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.372657113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131462024-02-27T21:49:07.003Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373047197Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131472024-02-27T21:49:07.003Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373213737Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131482024-02-27T21:49:07.003Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373262471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131492024-02-27T21:49:07.003Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373298837Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131502024-02-27T21:49:07.003Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373315115Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131512024-02-27T21:49:07.003Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373330363Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131522024-02-27T21:49:07.003Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373343892Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131532024-02-27T21:49:07.003Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373486575Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131542024-02-27T21:49:07.003Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373523671Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131552024-02-27T21:49:07.003Z{"msg":"[0]R flush_numbers[0..12]: [220, 213, 221, 221, 220, 220, 212, 203, 221, 219, 208, 218]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373540778Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131562024-02-27T21:49:07.003Z{"msg":"[0]R generation[0..12]: [120, 116, 120, 120, 120, 120, 115, 108, 120, 119, 113, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373562916Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131572024-02-27T21:49:07.003Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373577874Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131582024-02-27T21:49:07.003Z{"msg":"[1]R flush_numbers[0..12]: [220, 213, 221, 221, 220, 220, 212, 203, 221, 219, 208, 218]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373599462Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131592024-02-27T21:49:07.003Z{"msg":"[1]R generation[0..12]: [120, 116, 120, 120, 120, 120, 115, 108, 120, 119, 113, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.37361674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131602024-02-27T21:49:07.004Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373651545Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131612024-02-27T21:49:07.004Z{"msg":"[2]R flush_numbers[0..12]: [220, 213, 221, 221, 220, 220, 212, 203, 221, 219, 208, 218]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
131622024-02-27T21:49:07.004Zlevel":30,"time":"2024-02-27T21:49:05.373676173Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131632024-02-27T21:49:07.004Z{"msg":"[2]R generation[0..12]: [120, 116, 120, 120, 120, 120, 115, 108, 120, 119, 113, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.3737001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131642024-02-27T21:49:07.004Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373714838Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131652024-02-27T21:49:07.004Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373728507Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131662024-02-27T21:49:07.008Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.37378499Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131672024-02-27T21:49:07.008Z{"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373824925Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131682024-02-27T21:49:07.008Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373858961Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"":"downstairs","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131692024-02-27T21:49:07.008Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373889308Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131702024-02-27T21:49:07.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373910115Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"0","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131712024-02-27T21:49:07.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373947301Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"1","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131722024-02-27T21:49:07.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.373985656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"client":"2","":"downstairs","session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131732024-02-27T21:49:07.008Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.374024872Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131742024-02-27T21:49:07.008Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.37403895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131752024-02-27T21:49:07.008Z{"msg":"c3e280fe-4dc9-4542-8cc9-c38d288a7d1d is now active with session: 2916170c-255d-4ede-a503-7f620b8ce823","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.374052108Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131762024-02-27T21:49:07.009Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.374064817Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635,"session_id":"2916170c-255d-4ede-a503-7f620b8ce823"}
131772024-02-27T21:49:07.009Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:05.374085485Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635}
131782024-02-27T21:49:07.009ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131792024-02-27T21:49:07.009ZLoading write count information from file "/tmp/test_repair_verify.data"
131802024-02-27T21:49:07.009ZRun Repair workload
131812024-02-27T21:49:07.009Z01/30 Write block 95 len 24576 data: 10 9 8 7 10 11
131822024-02-27T21:49:07.009Z02/30 Read block 527 len 4096
131832024-02-27T21:49:07.009Z03/30 Flush
131842024-02-27T21:49:07.009Z04/30 Write block 413 len 32768 data: 8 9 10 11 14 13 11 7
131852024-02-27T21:49:07.009Z05/30 Write block 182 len 28672 data: 17 14 13 10 11 11 14
131862024-02-27T21:49:07.009Z06/30 Write block 268 len 20480 data: 8 9 9 10 11
131872024-02-27T21:49:07.009Z07/30 Write block 35 len 12288 data: 10 10 7
131882024-02-27T21:49:07.009Z08/30 Read block 20 len 8192
131892024-02-27T21:49:07.009Z09/30 Write block 423 len 8192 data: 6 7
131902024-02-27T21:49:07.009Z10/30 Read block 566 len 28672
131912024-02-27T21:49:07.013Z11/30 Flush
131922024-02-27T21:49:07.013Z12/30 Write block 102 len 28672 data: 10 10 8 10 12 15 14
131932024-02-27T21:49:07.013Z13/30 Write block 84 len 40960 data: 8 8 8 10 11 12 9 10 7 9
131942024-02-27T21:49:07.013Z14/30 Read block 526 len 12288
131952024-02-27T21:49:07.016Z15/30 Read block 38 len 36864
131962024-02-27T21:49:07.016Z16/30 Write block 195 len 40960 data: 6 6 6 5 6 7 6 6 6 5
131972024-02-27T21:49:07.016Z17/30 Write block 76 len 8192 data: 10 10
131982024-02-27T21:49:07.016Z18/30 Read block 424 len 12288
131992024-02-27T21:49:07.020Z19/30 Flush
132002024-02-27T21:49:07.023Z20/30 Write block 182 len 40960 data: 18 15 14 11 12 12 15 13 11 12
132012024-02-27T21:49:07.023Z21/30 Read block 514 len 40960
132022024-02-27T21:49:07.026Z22/30 Write block 585 len 36864 data: 12 12 10 10 11 10 8 9 9
132032024-02-27T21:49:07.026Z23/30 Flush
132042024-02-27T21:49:07.029Z24/30 Write block 408 len 40960 data: 6 6 6 5 7 9 10 11 12 15
132052024-02-27T21:49:07.029Z25/30 Write block 119 len 12288 data: 9 11 12
132062024-02-27T21:49:07.029Z26/30 Flush
132072024-02-27T21:49:07.032Z27/30 Write block 460 len 28672 data: 8 8 8 7 7 7 9
132082024-02-27T21:49:07.032Z28/30 Write block 76 len 8192 data: 11 11
132092024-02-27T21:49:07.036Z29/30 Read block 144 len 16384
132102024-02-27T21:49:07.036Z30/30 Write block 313 len 16384 data: 10 10 9 14
132112024-02-27T21:49:07.036Z----------------------------------------------------------------
132122024-02-27T21:49:07.036Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
132132024-02-27T21:49:07.036ZStates: Active Active Active
132142024-02-27T21:49:07.036ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132152024-02-27T21:49:07.036Z 1 Acked 1000 Write 6 Done Done Sent false
132162024-02-27T21:49:07.036Z 2 Acked 1001 Read 1 Done Done Sent false
132172024-02-27T21:49:07.036Z 3 Acked 1002 Flush 0 Done Done Sent false
132182024-02-27T21:49:07.036Z 4 Acked 1003 Write 8 Done Done Sent false
132192024-02-27T21:49:07.036Z 5 Acked 1004 Write 7 Done Done Sent false
132202024-02-27T21:49:07.036Z 6 Acked 1005 Write 5 Done Done Sent false
132212024-02-27T21:49:07.036Z 7 Acked 1006 Write 3 Done Done Sent false
132222024-02-27T21:49:07.036Z 8 Acked 1007 Read 2 Done Done Sent false
132232024-02-27T21:49:07.036Z 9 Acked 1008 Write 2 Done Done Sent false
132242024-02-27T21:49:07.036Z 10 Acked 1009 Read 7 Done Done Sent false
132252024-02-27T21:49:07.036Z 11 Acked 1010 Flush 0 Done Done Sent false
132262024-02-27T21:49:07.036Z 12 Acked 1011 Write 7 Done Done Sent false
132272024-02-27T21:49:07.036Z 13 Acked 1012 Write 10 Done Done Sent false
132282024-02-27T21:49:07.037Z 14 Acked 1013 Read 3 Done Done Sent false
132292024-02-27T21:49:07.037Z 15 Acked 1014 Read 9 Done Done Sent false
132302024-02-27T21:49:07.037Z 16 Acked 1015 Write 10 Done Done Sent false
132312024-02-27T21:49:07.037Z 17 Acked 1016 Write 2 Done Done Sent false
132322024-02-27T21:49:07.037Z 18 Acked 1017 Read 3 Done Done Sent false
132332024-02-27T21:49:07.037Z 19 Acked 1018 Flush 0 Done Done Sent false
132342024-02-27T21:49:07.037Z 20 Acked 1019 Write 10 Done Done Sent false
132352024-02-27T21:49:07.037Z 21 Acked 1020 Read 10 Done Done Sent false
132362024-02-27T21:49:07.037Z 22 Acked 1021 Write 9 Done Done Sent false
132372024-02-27T21:49:07.037Z 23 Acked 1022 Flush 0 Done Done Sent false
132382024-02-27T21:49:07.037Z 24 Acked 1023 Write 10 Done Done Sent false
132392024-02-27T21:49:07.037Z 25 Acked 1024 Write 3 Done Done Sent false
132402024-02-27T21:49:07.037Z 26 Acked 1025 Flush 0 Done Done Sent false
132412024-02-27T21:49:07.037Z 27 Acked 1026 Write 7 Done Done Sent false
132422024-02-27T21:49:07.037Z 28 Acked 1027 Write 2 Done Done Sent false
132432024-02-27T21:49:07.037Z 29 Acked 1028 Read 4 Done Done Sent false
132442024-02-27T21:49:07.037Z 30 Acked 1029 Write 4 Sent Sent Sent false
132452024-02-27T21:49:07.037Z STATES DS:0 DS:1 DS:2 TOTAL
132462024-02-27T21:49:07.037Z New 0 0 0 0
132472024-02-27T21:49:07.037Z Sent 1 1 30 32
132482024-02-27T21:49:07.037Z Done 29 29 0 58
132492024-02-27T21:49:07.037Z Skipped 0 0 0 0
132502024-02-27T21:49:07.037Z Error 0 0 0 0
132512024-02-27T21:49:07.037ZLast Flush: 1025 1025 0
132522024-02-27T21:49:07.038ZDownstairs last five completed:
132532024-02-27T21:49:07.038ZUpstairs last five completed: 30 29 28 27 26
132542024-02-27T21:49:07.038Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:05.410958871Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1635}
132552024-02-27T21:49:07.038ZWrote out file "/tmp/test_repair_verify.data"
132562024-02-27T21:49:07.041Z
132572024-02-27T21:49:07.044ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132582024-02-27T21:49:07.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132592024-02-27T21:49:07.057Z 0 000-019 120 120 120 220 220 220 F F F
132602024-02-27T21:49:07.057Z 1 020-039 122 122 116 224 224 213 F F F <---
132612024-02-27T21:49:07.057Z 2 040-059 120 120 120 221 221 221 F F F
132622024-02-27T21:49:07.057Z 3 060-079 122 122 120 225 225 221 T T F <---
132632024-02-27T21:49:07.057Z 4 080-099 122 122 120 225 225 220 F F F <---
132642024-02-27T21:49:07.057Z 5 100-119 122 122 120 227 227 220 F F F <---
132652024-02-27T21:49:07.057Z 6 120-139 122 122 115 227 227 212 F F F <---
132662024-02-27T21:49:07.057Z 7 140-159 108 108 108 203 203 203 F F F
132672024-02-27T21:49:07.057Z 8 160-179 120 120 120 221 221 221 F F F
132682024-02-27T21:49:07.057Z 9 180-199 122 122 119 226 226 219 F F F <---
132692024-02-27T21:49:07.057Z 10 200-219 122 122 113 225 225 208 F F F <---
132702024-02-27T21:49:07.064Z 11 220-239 118 118 118 218 218 218 F F F
132712024-02-27T21:49:07.064Z 12 240-259 119 119 119 219 219 219 F F F
132722024-02-27T21:49:07.064Z 13 260-279 122 122 115 224 224 212 F F F <---
132732024-02-27T21:49:07.064Z 14 280-299 120 120 120 220 220 220 F F F
132742024-02-27T21:49:07.064Z 15 300-319 111 111 111 206 206 206 T T F <---
132752024-02-27T21:49:07.064Z 16 320-339 121 121 121 222 222 222 F F F
132762024-02-27T21:49:07.064Z 17 340-359 119 119 119 219 219 219 F F F
132772024-02-27T21:49:07.064Z 18 360-379 107 107 107 202 202 202 F F F
132782024-02-27T21:49:07.064Z 19 380-399 120 120 120 220 220 220 F F F
132792024-02-27T21:49:07.064Z 20 400-419 122 122 115 227 227 212 F F F <---
132802024-02-27T21:49:07.064Z 21 420-439 122 122 116 224 224 215 F F F <---
132812024-02-27T21:49:07.064Z 22 440-459 113 113 113 208 208 208 F F F
132822024-02-27T21:49:07.064Z 23 460-479 121 121 121 222 222 222 T T F <---
132832024-02-27T21:49:07.065Z 24 480-499 108 108 108 203 203 203 F F F
132842024-02-27T21:49:07.065Z 25 500-519 120 120 120 220 220 220 F F F
132852024-02-27T21:49:07.065Z 26 520-539 120 120 120 220 220 220 F F F
132862024-02-27T21:49:07.065Z 27 540-559 115 115 115 212 212 212 F F F
132872024-02-27T21:49:07.065Z 28 560-579 118 118 118 218 218 218 F F F
132882024-02-27T21:49:07.065Z 29 580-599 122 122 116 226 226 213 F F F <---
132892024-02-27T21:49:07.065ZMax gen: 122, Max flush: 227
132902024-02-27T21:49:07.065ZError: Difference in extent metadata found!
132912024-02-27T21:49:07.065ZA Difference in extent metadata is expected here
132922024-02-27T21:49:07.065ZOn loop 61
132932024-02-27T21:49:07.065Z
132942024-02-27T21:49:07.068ZVerifying data now
132952024-02-27T21:49:11.718ZNo change after verify
132962024-02-27T21:49:11.718ZLoop: 61 Downstairs dump after verify (and repair):
132972024-02-27T21:49:11.736ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132982024-02-27T21:49:11.736Z 0 000-019 120 120 120 220 220 220 F F F
132992024-02-27T21:49:11.736Z 1 020-039 122 122 122 224 224 224 F F F
133002024-02-27T21:49:11.736Z 2 040-059 120 120 120 221 221 221 F F F
133012024-02-27T21:49:11.736Z 3 060-079 123 123 123 228 228 228 F F F
133022024-02-27T21:49:11.736Z 4 080-099 122 122 122 225 225 225 F F F
133032024-02-27T21:49:11.736Z 5 100-119 122 122 122 227 227 227 F F F
133042024-02-27T21:49:11.736Z 6 120-139 122 122 122 227 227 227 F F F
133052024-02-27T21:49:11.736Z 7 140-159 108 108 108 203 203 203 F F F
133062024-02-27T21:49:11.736Z 8 160-179 120 120 120 221 221 221 F F F
133072024-02-27T21:49:11.736Z 9 180-199 122 122 122 226 226 226 F F F
133082024-02-27T21:49:11.736Z 10 200-219 122 122 122 225 225 225 F F F
133092024-02-27T21:49:11.750Z 11 220-239 118 118 118 218 218 218 F F F
133102024-02-27T21:49:11.750Z 12 240-259 119 119 119 219 219 219 F F F
133112024-02-27T21:49:11.750Z 13 260-279 122 122 122 224 224 224 F F F
133122024-02-27T21:49:11.750Z 14 280-299 120 120 120 220 220 220 F F F
133132024-02-27T21:49:11.750Z 15 300-319 123 123 123 228 228 228 F F F
133142024-02-27T21:49:11.750Z 16 320-339 121 121 121 222 222 222 F F F
133152024-02-27T21:49:11.750Z 17 340-359 119 119 119 219 219 219 F F F
133162024-02-27T21:49:11.750Z 18 360-379 107 107 107 202 202 202 F F F
133172024-02-27T21:49:11.750Z 19 380-399 120 120 120 220 220 220 F F F
133182024-02-27T21:49:11.750Z 20 400-419 122 122 122 227 227 227 F F F
133192024-02-27T21:49:11.750Z 21 420-439 122 122 122 224 224 224 F F F
133202024-02-27T21:49:11.750Z 22 440-459 113 113 113 208 208 208 F F F
133212024-02-27T21:49:11.750Z 23 460-479 123 123 123 228 228 228 F F F
133222024-02-27T21:49:11.750Z 24 480-499 108 108 108 203 203 203 F F F
133232024-02-27T21:49:11.750Z 25 500-519 120 120 120 220 220 220 F F F
133242024-02-27T21:49:11.750Z 26 520-539 120 120 120 220 220 220 F F F
133252024-02-27T21:49:11.751Z 27 540-559 115 115 115 212 212 212 F F F
133262024-02-27T21:49:11.751Z 28 560-579 118 118 118 218 218 218 F F F
133272024-02-27T21:49:11.751Z 29 580-599 122 122 122 226 226 226 F F F
133282024-02-27T21:49:11.751ZMax gen: 123, Max flush: 228
133292024-02-27T21:49:11.751Z
133302024-02-27T21:49:11.751ZBegin loop 62
133312024-02-27T21:49:11.751ZDownstairs to restart: 2
133322024-02-27T21:49:11.760Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.125715878Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643}
133332024-02-27T21:49:11.760ZCrucible runtime is spawned
133342024-02-27T21:49:11.760Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126144398Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643}
133352024-02-27T21:49:11.760Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126184443Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643}
133362024-02-27T21:49:11.760Z{"msg":"Crucible 4dc5ffe9-7408-418d-bc49-5d8f3a0e30a8 has session id: 461b44dc-77f1-49f0-9af6-4300253fa404","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126201431Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133372024-02-27T21:49:11.761Z{"msg":"Upstairs opts: Upstairs UUID: 4dc5ffe9-7408-418d-bc49-5d8f3a0e30a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126241366Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133382024-02-27T21:49:11.761Z{"msg":"Crucible stats registered with UUID: 4dc5ffe9-7408-418d-bc49-5d8f3a0e30a8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126276722Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133392024-02-27T21:49:11.761Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.12629507Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643}
133402024-02-27T21:49:11.761Z{"msg":"4dc5ffe9-7408-418d-bc49-5d8f3a0e30a8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126315397Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133412024-02-27T21:49:11.761Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126347014Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133422024-02-27T21:49:11.761Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126386619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133432024-02-27T21:49:11.761Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126435283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133442024-02-27T21:49:11.761Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.12646721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133452024-02-27T21:49:11.761Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126493507Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133462024-02-27T21:49:11.761Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126521773Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133472024-02-27T21:49:11.761Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126557969Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133482024-02-27T21:49:11.761Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126625001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133492024-02-27T21:49:11.761Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126660177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133502024-02-27T21:49:11.761Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126704662Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133512024-02-27T21:49:11.761Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126746147Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133522024-02-27T21:49:11.761Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.126794681Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133532024-02-27T21:49:11.761Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.127519826Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133542024-02-27T21:49:11.761Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.127557821Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133552024-02-27T21:49:11.761Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.127576209Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133562024-02-27T21:49:11.761Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.127616394Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133572024-02-27T21:49:11.761Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.128041564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133582024-02-27T21:49:11.761Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.128085779Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133592024-02-27T21:49:11.761Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.128126074Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133602024-02-27T21:49:11.762Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.12816416Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133612024-02-27T21:49:11.762Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.128712975Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133622024-02-27T21:49:11.762Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.12875675Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133632024-02-27T21:49:11.762Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.128772788Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133642024-02-27T21:49:11.762Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.128859628Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133652024-02-27T21:49:11.762Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.128895064Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133662024-02-27T21:49:11.762Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.128920771Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133672024-02-27T21:49:11.762Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.128949637Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133682024-02-27T21:49:11.762Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129205137Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133692024-02-27T21:49:11.762Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129243003Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133702024-02-27T21:49:11.762Z{"msg":"[0]R flush_numbers[0..12]: [220, 224, 221, 228, 225, 227, 227, 203, 221, 226, 225, 218]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129260881Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133712024-02-27T21:49:11.762Z{"msg":"[0]R generation[0..12]: [120, 122, 120, 123, 122, 122, 122, 108, 120, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129283788Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133722024-02-27T21:49:11.762Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129299496Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133732024-02-27T21:49:11.762Z{"msg":"[1]R flush_numbers[0..12]: [220, 224, 221, 228, 225, 227, 227, 203, 221, 226, 225, 218]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129320674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133742024-02-27T21:49:11.762Z{"msg":"[1]R generation[0..12]: [120, 122, 120, 123, 122, 122, 122, 108, 120, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129335352Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133752024-02-27T21:49:11.762Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.12934978Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133762024-02-27T21:49:11.762Z{"msg":"[2]R flush_numbers[0..12]: [220, 224, 221, 228, 225, 227, 227, 203, 221, 226, 225, 218]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129375297Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133772024-02-27T21:49:11.762Z{"msg":"[2]R generation[0..12]: [120, 122, 120, 123, 122, 122, 122, 108, 120, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129392535Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133782024-02-27T21:49:11.762ZWait for a query_work_queue command to finish before sending IO
133792024-02-27T21:49:11.762Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129417272Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133802024-02-27T21:49:11.762Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.12943103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133812024-02-27T21:49:11.762Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129457458Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133822024-02-27T21:49:11.762Z{"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129470846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133832024-02-27T21:49:11.762Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129484804Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"":"downstairs","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133842024-02-27T21:49:11.762Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129505652Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133852024-02-27T21:49:11.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.12951848Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"0","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133862024-02-27T21:49:11.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129545287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"1","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133872024-02-27T21:49:11.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129565965Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"client":"2","":"downstairs","session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133882024-02-27T21:49:11.763Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129579833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133892024-02-27T21:49:11.763Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.12960625Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133902024-02-27T21:49:11.763Z{"msg":"4dc5ffe9-7408-418d-bc49-5d8f3a0e30a8 is now active with session: 461b44dc-77f1-49f0-9af6-4300253fa404","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129624088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133912024-02-27T21:49:11.763Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129644496Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643,"session_id":"461b44dc-77f1-49f0-9af6-4300253fa404"}
133922024-02-27T21:49:11.763Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:10.129657354Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643}
133932024-02-27T21:49:11.763ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133942024-02-27T21:49:11.763ZLoading write count information from file "/tmp/test_repair_verify.data"
133952024-02-27T21:49:11.763ZRun Repair workload
133962024-02-27T21:49:11.768Z01/30 Write block 251 len 24576 data: 11 11 11 9 13 13
133972024-02-27T21:49:11.768Z02/30 Write block 2 len 28672 data: 6 6 8 7 9 10 11
133982024-02-27T21:49:11.768Z03/30 Read block 139 len 40960
133992024-02-27T21:49:11.768Z04/30 Read block 112 len 4096
134002024-02-27T21:49:11.768Z05/30 Write block 4 len 16384 data: 9 8 10 11
134012024-02-27T21:49:11.768Z06/30 Read block 559 len 28672
134022024-02-27T21:49:11.768Z07/30 Write block 76 len 28672 data: 12 12 9 9 8 8 7
134032024-02-27T21:49:11.768Z08/30 Read block 566 len 4096
134042024-02-27T21:49:11.768Z09/30 Flush
134052024-02-27T21:49:11.768Z10/30 Flush
134062024-02-27T21:49:11.768Z11/30 Write block 314 len 32768 data: 11 10 15 14 13 15 15 13
134072024-02-27T21:49:11.768Z12/30 Write block 476 len 8192 data: 10 8
134082024-02-27T21:49:11.768Z13/30 Write block 128 len 8192 data: 12 10
134092024-02-27T21:49:11.771Z14/30 Read block 416 len 32768
134102024-02-27T21:49:11.771Z15/30 Write block 559 len 40960 data: 14 15 16 13 12 9 7 6 5 8
134112024-02-27T21:49:11.771Z16/30 Write block 44 len 36864 data: 13 15 11 11 10 10 11 11 11
134122024-02-27T21:49:11.771Z17/30 Write block 157 len 36864 data: 10 10 10 9 10 9 9 8 7
134132024-02-27T21:49:11.772Z18/30 Write block 234 len 20480 data: 16 16 16 10 10
134142024-02-27T21:49:11.775Z19/30 Flush
134152024-02-27T21:49:11.778Z20/30 Write block 408 len 24576 data: 7 7 7 6 8 10
134162024-02-27T21:49:11.778Z21/30 Read block 158 len 28672
134172024-02-27T21:49:11.784Z22/30 Write block 261 len 32768 data: 8 8 7 8 8 6 7 9
134182024-02-27T21:49:11.784Z23/30 Write block 52 len 12288 data: 12 11 10
134192024-02-27T21:49:11.784Z24/30 Write block 66 len 8192 data: 7 7
134202024-02-27T21:49:11.784Z25/30 Read block 504 len 16384
134212024-02-27T21:49:11.784Z26/30 Flush
134222024-02-27T21:49:11.784Z27/30 Write block 259 len 28672 data: 12 11 9 9 8 9 9
134232024-02-27T21:49:11.788Z28/30 Write block 382 len 36864 data: 8 8 9 9 8 10 9 10 10
134242024-02-27T21:49:11.788Z29/30 Read block 416 len 24576
134252024-02-27T21:49:11.789Z30/30 Write block 279 len 12288 data: 5 4 3
134262024-02-27T21:49:11.789Z----------------------------------------------------------------
134272024-02-27T21:49:11.789Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
134282024-02-27T21:49:11.789ZStates: Active Active Active
134292024-02-27T21:49:11.789ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
134302024-02-27T21:49:11.789Z 1 Acked 1000 Write 6 Done Done Done false
134312024-02-27T21:49:11.789Z 2 Acked 1001 Write 7 Done Done Done false
134322024-02-27T21:49:11.789Z 3 Acked 1002 Read 10 Done Done Sent false
134332024-02-27T21:49:11.789Z 4 Acked 1003 Read 1 Done Done Sent false
134342024-02-27T21:49:11.789Z 5 Acked 1004 Write 4 Done Done Sent false
134352024-02-27T21:49:11.789Z 6 Acked 1005 Read 7 Done Done Sent false
134362024-02-27T21:49:11.789Z 7 Acked 1006 Write 7 Done Done Sent false
134372024-02-27T21:49:11.789Z 8 Acked 1007 Read 1 Done Done Sent false
134382024-02-27T21:49:11.789Z 9 Acked 1008 Flush 0 Done Done Sent false
134392024-02-27T21:49:11.789Z 10 Acked 1009 Flush 0 Done Done Sent false
134402024-02-27T21:49:11.789Z 11 Acked 1010 Write 8 Done Done Sent false
134412024-02-27T21:49:11.789Z 12 Acked 1011 Write 2 Done Done Sent false
134422024-02-27T21:49:11.789Z 13 Acked 1012 Write 2 Done Done Sent false
134432024-02-27T21:49:11.789Z 14 Acked 1013 Read 8 Done Done Sent false
134442024-02-27T21:49:11.789Z 15 Acked 1014 Write 10 Done Done Sent false
134452024-02-27T21:49:11.789Z 16 Acked 1015 Write 9 Done Done Sent false
134462024-02-27T21:49:11.789Z 17 Acked 1016 Write 9 Done Done Sent false
134472024-02-27T21:49:11.789Z 18 Acked 1017 Write 5 Done Done Sent false
134482024-02-27T21:49:11.789Z 19 Acked 1018 Flush 0 Done Done Sent false
134492024-02-27T21:49:11.790Z 20 Acked 1019 Write 6 Done Done Sent false
134502024-02-27T21:49:11.790Z 21 Acked 1020 Read 7 Done Done Sent false
134512024-02-27T21:49:11.790Z 22 Acked 1021 Write 8 Done Done Sent false
134522024-02-27T21:49:11.790Z 23 Acked 1022 Write 3 Done Done Sent false
134532024-02-27T21:49:11.790Z 24 Acked 1023 Write 2 Done Done Sent false
134542024-02-27T21:49:11.790Z 25 Acked 1024 Read 4 Done Done Sent false
134552024-02-27T21:49:11.790Z 26 Acked 1025 Flush 0 Done Done Sent false
134562024-02-27T21:49:11.790Z 27 Acked 1026 Write 7 Done Done Sent false
134572024-02-27T21:49:11.790Z 28 Acked 1027 Write 9 Done Done Sent false
134582024-02-27T21:49:11.790Z 29 Acked 1028 Read 6 Done Done Sent false
134592024-02-27T21:49:11.790Z 30 Acked 1029 Write 3 Sent Sent Sent false
134602024-02-27T21:49:11.790Z STATES DS:0 DS:1 DS:2 TOTAL
134612024-02-27T21:49:11.790Z New 0 0 0 0
134622024-02-27T21:49:11.790Z Sent 1 1 28 30
134632024-02-27T21:49:11.790Z Done 29 29 2 60
134642024-02-27T21:49:11.790Z Skipped 0 0 0 0
134652024-02-27T21:49:11.790Z Error 0 0 0 0
134662024-02-27T21:49:11.790ZLast Flush: 1025 1025 0
134672024-02-27T21:49:11.790ZDownstairs last five completed:
134682024-02-27T21:49:11.790ZUpstairs last five completed: 30 29 28 27 26
134692024-02-27T21:49:11.790Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:10.16356025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1643}
134702024-02-27T21:49:11.790ZWrote out file "/tmp/test_repair_verify.data"
134712024-02-27T21:49:11.794Z
134722024-02-27T21:49:11.995ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134732024-02-27T21:49:12.006ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134742024-02-27T21:49:12.006Z 0 000-019 124 124 120 229 229 220 F F T <---
134752024-02-27T21:49:12.006Z 1 020-039 122 122 122 224 224 224 F F F
134762024-02-27T21:49:12.006Z 2 040-059 124 124 120 232 232 221 F F F <---
134772024-02-27T21:49:12.006Z 3 060-079 124 124 123 232 232 228 F F F <---
134782024-02-27T21:49:12.006Z 4 080-099 124 124 122 229 229 225 F F F <---
134792024-02-27T21:49:12.007Z 5 100-119 122 122 122 227 227 227 F F F
134802024-02-27T21:49:12.007Z 6 120-139 124 124 122 231 231 227 F F F <---
134812024-02-27T21:49:12.007Z 7 140-159 124 124 108 231 231 203 F F F <---
134822024-02-27T21:49:12.007Z 8 160-179 124 124 120 231 231 221 F F F <---
134832024-02-27T21:49:12.007Z 9 180-199 122 122 122 226 226 226 F F F
134842024-02-27T21:49:12.007Z 10 200-219 122 122 122 225 225 225 F F F
134852024-02-27T21:49:12.007Z 11 220-239 124 124 118 231 231 218 F F F <---
134862024-02-27T21:49:12.007Z 12 240-259 124 124 119 229 229 219 T T T <---
134872024-02-27T21:49:12.007Z 13 260-279 124 124 122 232 232 224 T T F <---
134882024-02-27T21:49:12.007Z 14 280-299 120 120 120 220 220 220 T T F <---
134892024-02-27T21:49:12.007Z 15 300-319 124 124 123 231 231 228 F F F <---
134902024-02-27T21:49:12.007Z 16 320-339 124 124 121 231 231 222 F F F <---
134912024-02-27T21:49:12.007Z 17 340-359 119 119 119 219 219 219 F F F
134922024-02-27T21:49:12.007Z 18 360-379 107 107 107 202 202 202 F F F
134932024-02-27T21:49:12.007Z 19 380-399 120 120 120 220 220 220 T T F <---
134942024-02-27T21:49:12.007Z 20 400-419 124 124 122 232 232 227 F F F <---
134952024-02-27T21:49:12.007Z 21 420-439 122 122 122 224 224 224 F F F
134962024-02-27T21:49:12.007Z 22 440-459 113 113 113 208 208 208 F F F
134972024-02-27T21:49:12.007Z 23 460-479 124 124 123 231 231 228 F F F <---
134982024-02-27T21:49:12.007Z 24 480-499 108 108 108 203 203 203 F F F
134992024-02-27T21:49:12.007Z 25 500-519 120 120 120 220 220 220 F F F
135002024-02-27T21:49:12.008Z 26 520-539 120 120 120 220 220 220 F F F
135012024-02-27T21:49:12.008Z 27 540-559 124 124 115 231 231 212 F F F <---
135022024-02-27T21:49:12.008Z 28 560-579 124 124 118 231 231 218 F F F <---
135032024-02-27T21:49:12.008Z 29 580-599 122 122 122 226 226 226 F F F
135042024-02-27T21:49:12.008ZMax gen: 124, Max flush: 232
135052024-02-27T21:49:12.008ZError: Difference in extent metadata found!
135062024-02-27T21:49:12.008ZA Difference in extent metadata is expected here
135072024-02-27T21:49:12.008ZOn loop 62
135082024-02-27T21:49:12.008Z
135092024-02-27T21:49:12.008ZVerifying data now
135102024-02-27T21:49:16.677ZNo change after verify
135112024-02-27T21:49:16.677ZLoop: 62 Downstairs dump after verify (and repair):
135122024-02-27T21:49:16.691ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135132024-02-27T21:49:16.691Z 0 000-019 124 124 124 229 229 229 F F F
135142024-02-27T21:49:16.691Z 1 020-039 122 122 122 224 224 224 F F F
135152024-02-27T21:49:16.691Z 2 040-059 124 124 124 232 232 232 F F F
135162024-02-27T21:49:16.691Z 3 060-079 124 124 124 232 232 232 F F F
135172024-02-27T21:49:16.691Z 4 080-099 124 124 124 229 229 229 F F F
135182024-02-27T21:49:16.691Z 5 100-119 122 122 122 227 227 227 F F F
135192024-02-27T21:49:16.695Z 6 120-139 124 124 124 231 231 231 F F F
135202024-02-27T21:49:16.695Z 7 140-159 124 124 124 231 231 231 F F F
135212024-02-27T21:49:16.695Z 8 160-179 124 124 124 231 231 231 F F F
135222024-02-27T21:49:16.695Z 9 180-199 122 122 122 226 226 226 F F F
135232024-02-27T21:49:16.695Z 10 200-219 122 122 122 225 225 225 F F F
135242024-02-27T21:49:16.695Z 11 220-239 124 124 124 231 231 231 F F F
135252024-02-27T21:49:16.695Z 12 240-259 125 125 125 233 233 233 F F F
135262024-02-27T21:49:16.695Z 13 260-279 125 125 125 233 233 233 F F F
135272024-02-27T21:49:16.695Z 14 280-299 125 125 125 233 233 233 F F F
135282024-02-27T21:49:16.695Z 15 300-319 124 124 124 231 231 231 F F F
135292024-02-27T21:49:16.695Z 16 320-339 124 124 124 231 231 231 F F F
135302024-02-27T21:49:16.695Z 17 340-359 119 119 119 219 219 219 F F F
135312024-02-27T21:49:16.695Z 18 360-379 107 107 107 202 202 202 F F F
135322024-02-27T21:49:16.695Z 19 380-399 125 125 125 233 233 233 F F F
135332024-02-27T21:49:16.695Z 20 400-419 124 124 124 232 232 232 F F F
135342024-02-27T21:49:16.696Z 21 420-439 122 122 122 224 224 224 F F F
135352024-02-27T21:49:16.696Z 22 440-459 113 113 113 208 208 208 F F F
135362024-02-27T21:49:16.696Z 23 460-479 124 124 124 231 231 231 F F F
135372024-02-27T21:49:16.696Z 24 480-499 108 108 108 203 203 203 F F F
135382024-02-27T21:49:16.696Z 25 500-519 120 120 120 220 220 220 F F F
135392024-02-27T21:49:16.696Z 26 520-539 120 120 120 220 220 220 F F F
135402024-02-27T21:49:16.696Z 27 540-559 124 124 124 231 231 231 F F F
135412024-02-27T21:49:16.696Z 28 560-579 124 124 124 231 231 231 F F F
135422024-02-27T21:49:16.696Z 29 580-599 122 122 122 226 226 226 F F F
135432024-02-27T21:49:16.696ZMax gen: 125, Max flush: 233
135442024-02-27T21:49:16.696Z
135452024-02-27T21:49:16.696ZBegin loop 63
135462024-02-27T21:49:16.696ZDownstairs to restart: 0
135472024-02-27T21:49:16.714Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.088979833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651}
135482024-02-27T21:49:16.714Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089279348Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651}
135492024-02-27T21:49:16.714Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089311814Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651}
135502024-02-27T21:49:16.714ZCrucible runtime is spawned
135512024-02-27T21:49:16.721Z{"msg":"Crucible 3aeddd74-e73e-4869-8dfb-4d0636465e3a has session id: e2fc2dc2-1008-40a5-b74b-ecf6466fccd5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089364818Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135522024-02-27T21:49:16.721Z{"msg":"Upstairs opts: Upstairs UUID: 3aeddd74-e73e-4869-8dfb-4d0636465e3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089442159Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135532024-02-27T21:49:16.721Z{"msg":"Crucible stats registered with UUID: 3aeddd74-e73e-4869-8dfb-4d0636465e3a","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089479565Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135542024-02-27T21:49:16.721Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089497382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651}
135552024-02-27T21:49:16.721Z{"msg":"3aeddd74-e73e-4869-8dfb-4d0636465e3a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089532028Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135562024-02-27T21:49:16.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089558235Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135572024-02-27T21:49:16.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089593641Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135582024-02-27T21:49:16.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089630287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135592024-02-27T21:49:16.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089655484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135602024-02-27T21:49:16.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089696109Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135612024-02-27T21:49:16.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089723776Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135622024-02-27T21:49:16.721Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089756822Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135632024-02-27T21:49:16.721Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.089782969Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135642024-02-27T21:49:16.721Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.09002525Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135652024-02-27T21:49:16.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.090066515Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135662024-02-27T21:49:16.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.090084283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135672024-02-27T21:49:16.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.090098841Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135682024-02-27T21:49:16.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.090816857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135692024-02-27T21:49:16.721Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.090853812Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135702024-02-27T21:49:16.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.090869311Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135712024-02-27T21:49:16.722Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.090892218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135722024-02-27T21:49:16.722Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.090917335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135732024-02-27T21:49:16.722Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.090942832Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135742024-02-27T21:49:16.722Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.091317308Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135752024-02-27T21:49:16.722Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.091361743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135762024-02-27T21:49:16.722Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.091550981Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135772024-02-27T21:49:16.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.091935355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135782024-02-27T21:49:16.722Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.091993208Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135792024-02-27T21:49:16.722Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092031734Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135802024-02-27T21:49:16.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092051331Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135812024-02-27T21:49:16.722Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092094526Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135822024-02-27T21:49:16.722Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092121633Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135832024-02-27T21:49:16.722Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092152759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135842024-02-27T21:49:16.722Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092177577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135852024-02-27T21:49:16.722Z{"msg":"[0]R flush_numbers[0..12]: [229, 224, 232, 232, 229, 227, 231, 231, 231, 226, 225, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092206303Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135862024-02-27T21:49:16.722Z{"msg":"[0]R generation[0..12]: [124, 122, 124, 124, 124, 122, 124, 124, 124, 122, 122, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.09223229Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135872024-02-27T21:49:16.722Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
135882024-02-27T21:49:16.722Z,"time":"2024-02-27T21:49:15.092263377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135892024-02-27T21:49:16.722Z{"msg":"[1]R flush_numbers[0..12]: [229, 224, 232, 232, 229, 227, 231, 231, 231, 226, 225, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092293373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135902024-02-27T21:49:16.722Z{"msg":"[1]R generation[0..12]: [124, 122, 124, 124, 124, 122, 124, 124, 124, 122, 122, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092322379Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135912024-02-27T21:49:16.722Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092348336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135922024-02-27T21:49:16.722Z{"msg":"[2]R flush_numbers[0..12]: [229, 224, 232, 232, 229, 227, 231, 231, 231, 226, 225, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092371924Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135932024-02-27T21:49:16.722Z{"msg":"[2]R generation[0..12]: [124, 122, 124, 124, 124, 122, 124, 124, 124, 122, 122, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092410669Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135942024-02-27T21:49:16.722Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092444725Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135952024-02-27T21:49:16.722Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.09248668Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135962024-02-27T21:49:16.723Zdownstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135972024-02-27T21:49:16.723Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.09257315Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135982024-02-27T21:49:16.723Z{"msg":"Next flush: 234","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092618575Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
135992024-02-27T21:49:16.723Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092652911Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"":"downstairs","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
136002024-02-27T21:49:16.723Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092685347Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
136012024-02-27T21:49:16.723Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092715353Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"0","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
136022024-02-27T21:49:16.723Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092746809Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"1","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
136032024-02-27T21:49:16.723Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092777926Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"client":"2","":"downstairs","session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
136042024-02-27T21:49:16.723Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092808992Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
136052024-02-27T21:49:16.723Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092837699Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
136062024-02-27T21:49:16.723Z{"msg":"3aeddd74-e73e-4869-8dfb-4d0636465e3a is now active with session: e2fc2dc2-1008-40a5-b74b-ecf6466fccd5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092862086Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
136072024-02-27T21:49:16.723Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092891312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651,"session_id":"e2fc2dc2-1008-40a5-b74b-ecf6466fccd5"}
136082024-02-27T21:49:16.723Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:15.092919919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651}
136092024-02-27T21:49:16.727ZLoading write count information from file "/tmp/test_repair_verify.data"
136102024-02-27T21:49:16.727ZRun Repair workload
136112024-02-27T21:49:16.727Z01/30 Write block 191 len 20480 data: 13 11 9 8 7
136122024-02-27T21:49:16.727Z02/30 Write block 590 len 4096 data: 11
136132024-02-27T21:49:16.727Z03/30 Read block 383 len 24576
136142024-02-27T21:49:16.727Z04/30 Read block 295 len 16384
136152024-02-27T21:49:16.727Z05/30 Read block 556 len 32768
136162024-02-27T21:49:16.727Z06/30 Flush
136172024-02-27T21:49:16.730Z07/30 Write block 100 len 12288 data: 12 10 11
136182024-02-27T21:49:16.730Z08/30 Write block 28 len 24576 data: 8 9 11 10 9 10
136192024-02-27T21:49:16.730Z09/30 Read block 512 len 32768
136202024-02-27T21:49:16.730Z10/30 Read block 329 len 32768
136212024-02-27T21:49:16.730Z11/30 Read block 352 len 8192
136222024-02-27T21:49:16.734Z12/30 Read block 219 len 16384
136232024-02-27T21:49:16.734Z13/30 Flush
136242024-02-27T21:49:16.737Z14/30 Write block 366 len 12288 data: 10 10 10
136252024-02-27T21:49:16.737Z15/30 Flush
136262024-02-27T21:49:16.737Z16/30 Write block 300 len 20480 data: 8 8 10 8 11
136272024-02-27T21:49:16.740Z17/30 Write block 152 len 20480 data: 7 6 6 8 9
136282024-02-27T21:49:16.740Z18/30 Write block 343 len 16384 data: 11 11 9 9
136292024-02-27T21:49:16.740Z19/30 Read block 584 len 36864
136302024-02-27T21:49:16.740Z20/30 Write block 293 len 36864 data: 11 10 11 11 11 11 8 9 9
136312024-02-27T21:49:16.743Z21/30 Read block 219 len 24576
136322024-02-27T21:49:16.743Z22/30 Write block 380 len 16384 data: 5 6 9 9
136332024-02-27T21:49:16.744Z23/30 Write block 530 len 28672 data: 6 6 5 6 9 10 9
136342024-02-27T21:49:16.744Z24/30 Write block 111 len 8192 data: 12 10
136352024-02-27T21:49:16.748Z25/30 Read block 9 len 40960
136362024-02-27T21:49:16.748Z26/30 Write block 153 len 16384 data: 7 7 9 10
136372024-02-27T21:49:16.749Z27/30 Read block 332 len 28672
136382024-02-27T21:49:16.754Z28/30 Read block 137 len 8192
136392024-02-27T21:49:16.754Z29/30 Read block 67 len 12288
136402024-02-27T21:49:16.754Z30/30 Write block 160 len 28672 data: 10 11 10 10 9 8 4
136412024-02-27T21:49:16.754Z----------------------------------------------------------------
136422024-02-27T21:49:16.755Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
136432024-02-27T21:49:16.755ZStates: Active Active Active
136442024-02-27T21:49:16.755ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136452024-02-27T21:49:16.755Z 1 Acked 1000 Write 5 Sent Done Done false
136462024-02-27T21:49:16.755Z 2 Acked 1001 Write 1 Sent Done Done false
136472024-02-27T21:49:16.755Z 3 Acked 1002 Read 6 Sent Done Done false
136482024-02-27T21:49:16.755Z 4 Acked 1003 Read 4 Sent Done Done false
136492024-02-27T21:49:16.755Z 5 Acked 1004 Read 8 Sent Done Done false
136502024-02-27T21:49:16.755Z 6 Acked 1005 Flush 0 Sent Done Done false
136512024-02-27T21:49:16.755Z 7 Acked 1006 Write 3 Sent Done Done false
136522024-02-27T21:49:16.755Z 8 Acked 1007 Write 6 Sent Done Done false
136532024-02-27T21:49:16.755Z 9 Acked 1008 Read 8 Sent Done Done false
136542024-02-27T21:49:16.756Z 10 Acked 1009 Read 8 Sent Done Done false
136552024-02-27T21:49:16.756Z 11 Acked 1010 Read 2 Sent Done Done false
136562024-02-27T21:49:16.756Z 12 Acked 1011 Read 4 Sent Done Done false
136572024-02-27T21:49:16.756Z 13 Acked 1012 Flush 0 Sent Done Done false
136582024-02-27T21:49:16.756Z 14 Acked 1013 Write 3 Sent Done Done false
136592024-02-27T21:49:16.756Z 15 Acked 1014 Flush 0 Sent Done Done false
136602024-02-27T21:49:16.756Z 16 Acked 1015 Write 5 Sent Done Done false
136612024-02-27T21:49:16.756Z 17 Acked 1016 Write 5 Sent Done Done false
136622024-02-27T21:49:16.756Z 18 Acked 1017 Write 4 Sent Done Done false
136632024-02-27T21:49:16.757Z 19 Acked 1018 Read 9 Sent Done Done false
136642024-02-27T21:49:16.757Z 20 Acked 1019 Write 9 Sent Done Done false
136652024-02-27T21:49:16.757Z 21 Acked 1020 Read 6 Sent Done Done false
136662024-02-27T21:49:16.757Z 22 Acked 1021 Write 4 Sent Done Done false
136672024-02-27T21:49:16.757Z 23 Acked 1022 Write 7 Sent Done Done false
136682024-02-27T21:49:16.757Z 24 Acked 1023 Write 2 Sent Done Done false
136692024-02-27T21:49:16.757Z 25 Acked 1024 Read 10 Sent Done Done false
136702024-02-27T21:49:16.757Z 26 Acked 1025 Write 4 Sent Done Done false
136712024-02-27T21:49:16.757Z 27 Acked 1026 Read 7 Sent Done Done false
136722024-02-27T21:49:16.757Z 28 Acked 1027 Read 2 Sent Done Done false
136732024-02-27T21:49:16.757Z 29 Acked 1028 Read 3 Sent Done Done false
136742024-02-27T21:49:16.757Z 30 Acked 1029 Write 7 Sent Sent Sent false
136752024-02-27T21:49:16.757Z STATES DS:0 DS:1 DS:2 TOTAL
136762024-02-27T21:49:16.757Z New 0 0 0 0
136772024-02-27T21:49:16.758Z Sent 30 1 1 32
136782024-02-27T21:49:16.758Z Done 0 29 29 58
136792024-02-27T21:49:16.758Z Skipped 0 0 0 0
136802024-02-27T21:49:16.758Z Error 0 0 0 0
136812024-02-27T21:49:16.758ZLast Flush: 0 1014 1014
136822024-02-27T21:49:16.758ZDownstairs last five completed:
136832024-02-27T21:49:16.758ZUpstairs last five completed: 30 29 28 27 26
136842024-02-27T21:49:16.758Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:15.124953656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1651}
136852024-02-27T21:49:16.758ZWrote out file "/tmp/test_repair_verify.data"
136862024-02-27T21:49:16.762Z
136872024-02-27T21:49:16.762ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136882024-02-27T21:49:16.775ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136892024-02-27T21:49:16.775Z 0 000-019 124 124 124 229 229 229 F F F
136902024-02-27T21:49:16.775Z 1 020-039 122 126 126 224 235 235 F F F <---
136912024-02-27T21:49:16.775Z 2 040-059 124 124 124 232 232 232 F F F
136922024-02-27T21:49:16.775Z 3 060-079 124 124 124 232 232 232 F F F
136932024-02-27T21:49:16.775Z 4 080-099 124 124 124 229 229 229 F F F
136942024-02-27T21:49:16.775Z 5 100-119 122 126 126 227 235 235 F T T <---
136952024-02-27T21:49:16.775Z 6 120-139 124 124 124 231 231 231 F F F
136962024-02-27T21:49:16.775Z 7 140-159 124 124 124 231 231 231 F T T <---
136972024-02-27T21:49:16.780Z 8 160-179 124 124 124 231 231 231 F T T <---
136982024-02-27T21:49:16.780Z 9 180-199 122 126 126 226 234 234 F F F <---
136992024-02-27T21:49:16.780Z 10 200-219 122 122 122 225 225 225 F F F
137002024-02-27T21:49:16.780Z 11 220-239 124 124 124 231 231 231 F F F
137012024-02-27T21:49:16.780Z 12 240-259 125 125 125 233 233 233 F F F
137022024-02-27T21:49:16.780Z 13 260-279 125 125 125 233 233 233 F F F
137032024-02-27T21:49:16.780Z 14 280-299 125 125 125 233 233 233 F T T <---
137042024-02-27T21:49:16.780Z 15 300-319 124 124 124 231 231 231 F T T <---
137052024-02-27T21:49:16.780Z 16 320-339 124 124 124 231 231 231 F F F
137062024-02-27T21:49:16.780Z 17 340-359 119 119 119 219 219 219 F T T <---
137072024-02-27T21:49:16.780Z 18 360-379 107 126 126 202 236 236 F F F <---
137082024-02-27T21:49:16.780Z 19 380-399 125 125 125 233 233 233 F T T <---
137092024-02-27T21:49:16.781Z 20 400-419 124 124 124 232 232 232 F F F
137102024-02-27T21:49:16.781Z 21 420-439 122 122 122 224 224 224 F F F
137112024-02-27T21:49:16.781Z 22 440-459 113 113 113 208 208 208 F F F
137122024-02-27T21:49:16.781Z 23 460-479 124 124 124 231 231 231 F F F
137132024-02-27T21:49:16.781Z 24 480-499 108 108 108 203 203 203 F F F
137142024-02-27T21:49:16.781Z 25 500-519 120 120 120 220 220 220 F F F
137152024-02-27T21:49:16.781Z 26 520-539 120 120 120 220 220 220 F T T <---
137162024-02-27T21:49:16.781Z 27 540-559 124 124 124 231 231 231 F F F
137172024-02-27T21:49:16.781Z 28 560-579 124 124 124 231 231 231 F F F
137182024-02-27T21:49:16.781Z 29 580-599 122 126 126 226 234 234 F F F <---
137192024-02-27T21:49:16.781ZMax gen: 126, Max flush: 236
137202024-02-27T21:49:16.781ZError: Difference in extent metadata found!
137212024-02-27T21:49:16.781ZA Difference in extent metadata is expected here
137222024-02-27T21:49:16.781ZOn loop 63
137232024-02-27T21:49:16.781Z
137242024-02-27T21:49:16.790ZVerifying data now
137252024-02-27T21:49:21.402ZNo change after verify
137262024-02-27T21:49:21.403ZLoop: 63 Downstairs dump after verify (and repair):
137272024-02-27T21:49:21.418ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137282024-02-27T21:49:21.418Z 0 000-019 124 124 124 229 229 229 F F F
137292024-02-27T21:49:21.418Z 1 020-039 126 126 126 235 235 235 F F F
137302024-02-27T21:49:21.418Z 2 040-059 124 124 124 232 232 232 F F F
137312024-02-27T21:49:21.418Z 3 060-079 124 124 124 232 232 232 F F F
137322024-02-27T21:49:21.418Z 4 080-099 124 124 124 229 229 229 F F F
137332024-02-27T21:49:21.427Z 5 100-119 127 127 127 237 237 237 F F F
137342024-02-27T21:49:21.427Z 6 120-139 124 124 124 231 231 231 F F F
137352024-02-27T21:49:21.427Z 7 140-159 127 127 127 237 237 237 F F F
137362024-02-27T21:49:21.427Z 8 160-179 127 127 127 237 237 237 F F F
137372024-02-27T21:49:21.427Z 9 180-199 126 126 126 234 234 234 F F F
137382024-02-27T21:49:21.427Z 10 200-219 122 122 122 225 225 225 F F F
137392024-02-27T21:49:21.427Z 11 220-239 124 124 124 231 231 231 F F F
137402024-02-27T21:49:21.427Z 12 240-259 125 125 125 233 233 233 F F F
137412024-02-27T21:49:21.427Z 13 260-279 125 125 125 233 233 233 F F F
137422024-02-27T21:49:21.427Z 14 280-299 127 127 127 237 237 237 F F F
137432024-02-27T21:49:21.427Z 15 300-319 127 127 127 237 237 237 F F F
137442024-02-27T21:49:21.427Z 16 320-339 124 124 124 231 231 231 F F F
137452024-02-27T21:49:21.427Z 17 340-359 127 127 127 237 237 237 F F F
137462024-02-27T21:49:21.427Z 18 360-379 126 126 126 236 236 236 F F F
137472024-02-27T21:49:21.427Z 19 380-399 127 127 127 237 237 237 F F F
137482024-02-27T21:49:21.428Z 20 400-419 124 124 124 232 232 232 F F F
137492024-02-27T21:49:21.428Z 21 420-439 122 122 122 224 224 224 F F F
137502024-02-27T21:49:21.428Z 22 440-459 113 113 113 208 208 208 F F F
137512024-02-27T21:49:21.428Z 23 460-479 124 124 124 231 231 231 F F F
137522024-02-27T21:49:21.428Z 24 480-499 108 108 108 203 203 203 F F F
137532024-02-27T21:49:21.428Z 25 500-519 120 120 120 220 220 220 F F F
137542024-02-27T21:49:21.428Z 26 520-539 127 127 127 237 237 237 F F F
137552024-02-27T21:49:21.428Z 27 540-559 124 124 124 231 231 231 F F F
137562024-02-27T21:49:21.428Z 28 560-579 124 124 124 231 231 231 F F F
137572024-02-27T21:49:21.428Z 29 580-599 126 126 126 234 234 234 F F F
137582024-02-27T21:49:21.428ZMax gen: 127, Max flush: 237
137592024-02-27T21:49:21.428Z
137602024-02-27T21:49:21.428ZBegin loop 64
137612024-02-27T21:49:21.428ZDownstairs to restart: 0
137622024-02-27T21:49:21.437Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.811830658Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659}
137632024-02-27T21:49:21.437Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812111865Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659}
137642024-02-27T21:49:21.445Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812142421Z",Crucible runtime is spawned
137652024-02-27T21:49:21.445Z"hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659}
137662024-02-27T21:49:21.445Z{"msg":"Crucible a076f8f6-337e-43cf-ada6-93b91a7959ce has session id: 64071142-ffa8-4bd0-94d8-956f450175ce","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812258258Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137672024-02-27T21:49:21.445Z{"msg":"Upstairs opts: Upstairs UUID: a076f8f6-337e-43cf-ada6-93b91a7959ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812354626Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137682024-02-27T21:49:21.445Z{"msg":"Crucible stats registered with UUID: a076f8f6-337e-43cf-ada6-93b91a7959ce","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812404321Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137692024-02-27T21:49:21.445Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812432227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659}
137702024-02-27T21:49:21.445Z{"msg":"a076f8f6-337e-43cf-ada6-93b91a7959ce active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812453185Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137712024-02-27T21:49:21.445Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812474442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137722024-02-27T21:49:21.445Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812504749Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137732024-02-27T21:49:21.445Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812532855Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137742024-02-27T21:49:21.445Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.81257643Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137752024-02-27T21:49:21.445Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812615676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137762024-02-27T21:49:21.445Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812643762Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137772024-02-27T21:49:21.445Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812678958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137782024-02-27T21:49:21.445Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812720503Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137792024-02-27T21:49:21.445Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812756999Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137802024-02-27T21:49:21.445Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812793955Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137812024-02-27T21:49:21.445Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812828631Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137822024-02-27T21:49:21.446Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.812857557Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137832024-02-27T21:49:21.446Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.813403613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137842024-02-27T21:49:21.446Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.813451967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137852024-02-27T21:49:21.446Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.813493552Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137862024-02-27T21:49:21.446Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.813545796Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137872024-02-27T21:49:21.446Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.81359151Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137882024-02-27T21:49:21.446Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.813613868Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137892024-02-27T21:49:21.446Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814041357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137902024-02-27T21:49:21.446Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814081453Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137912024-02-27T21:49:21.446Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814110949Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137922024-02-27T21:49:21.446Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814607461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137932024-02-27T21:49:21.446Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814644226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137942024-02-27T21:49:21.446Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814659415Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137952024-02-27T21:49:21.446Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814675343Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137962024-02-27T21:49:21.446Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.8146961Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137972024-02-27T21:49:21.446Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814719268Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137982024-02-27T21:49:21.446Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814733146Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
137992024-02-27T21:49:21.446Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814753534Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138002024-02-27T21:49:21.446Z{"msg":"[0]R flush_numbers[0..12]: [229, 235, 232, 232, 229, 237, 231, 237, 237, 234, 225, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814768482Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138012024-02-27T21:49:21.446Z{"msg":"[0]R generation[0..12]: [124, 126, 124, 124, 124, 127, 124, 127, 127, 126, 122, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814790069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138022024-02-27T21:49:21.446Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814805338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138032024-02-27T21:49:21.446Z{"msg":"[1]R flush_numbers[0..12]: [229, 235, 232, 232, 229, 237, 231, 237, 237, 234, 225, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814826935Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138042024-02-27T21:49:21.446Z{"msg":"[1]R generation[0..12]: [124, 126, 124, 124, 124, 127, 124, 127, 127, 126, 122, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814841323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138052024-02-27T21:49:21.446Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814862461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138062024-02-27T21:49:21.446Z{"msg":"[2]R flush_numbers[0..12]: [229, 235, 232, 232, 229, 237, 231, 237, 237, 234, 225, 231]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
138072024-02-27T21:49:21.447Ztime":"2024-02-27T21:49:19.814877499Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138082024-02-27T21:49:21.447Z{"msg":"[2]R generation[0..12]: [124, 126, 124, 124, 124, 127, 124, 127, 127, 126, 122, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814905236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138092024-02-27T21:49:21.447Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814926923Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138102024-02-27T21:49:21.447Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814940902Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138112024-02-27T21:49:21.447Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814960589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138122024-02-27T21:49:21.447Z{"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814973818Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138132024-02-27T21:49:21.447Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.814987496Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"":"downstairs","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138142024-02-27T21:49:21.447Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.815003564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138152024-02-27T21:49:21.447Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.815020342Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"0","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138162024-02-27T21:49:21.447Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.815034111Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"1","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138172024-02-27T21:49:21.447Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.815055058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"client":"2","":"downstairs","session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138182024-02-27T21:49:21.447Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.815068907Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138192024-02-27T21:49:21.447Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.815087824Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138202024-02-27T21:49:21.447Z{"msg":"a076f8f6-337e-43cf-ada6-93b91a7959ce is now active with session: 64071142-ffa8-4bd0-94d8-956f450175ce","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.815101103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138212024-02-27T21:49:21.447Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.81512139Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659,"session_id":"64071142-ffa8-4bd0-94d8-956f450175ce"}
138222024-02-27T21:49:21.447Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:19.815134619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659}
138232024-02-27T21:49:21.447ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138242024-02-27T21:49:21.447ZLoading write count information from file "/tmp/test_repair_verify.data"
138252024-02-27T21:49:21.447ZRun Repair workload
138262024-02-27T21:49:21.452Z01/30 Write block 384 len 40960 data: 10 10 9 11 10 11 11 13 11 12
138272024-02-27T21:49:21.452Z02/30 Read block 113 len 8192
138282024-02-27T21:49:21.452Z03/30 Write block 144 len 12288 data: 8 7 8
138292024-02-27T21:49:21.452Z04/30 Flush
138302024-02-27T21:49:21.453Z05/30 Flush
138312024-02-27T21:49:21.453Z06/30 Write block 571 len 36864 data: 8 8 9 7 7 7 8 10 7
138322024-02-27T21:49:21.453Z07/30 Write block 121 len 20480 data: 13 11 12 14 11
138332024-02-27T21:49:21.453Z08/30 Read block 357 len 40960
138342024-02-27T21:49:21.453Z09/30 Write block 155 len 24576 data: 10 11 11 11 11 11
138352024-02-27T21:49:21.453Z10/30 Write block 502 len 8192 data: 6 6
138362024-02-27T21:49:21.453Z11/30 Write block 204 len 36864 data: 6 4 4 5 7 10 10 10 11
138372024-02-27T21:49:21.459Z12/30 Write block 408 len 8192 data: 8 8
138382024-02-27T21:49:21.459Z13/30 Read block 84 len 36864
138392024-02-27T21:49:21.459Z14/30 Write block 166 len 8192 data: 5 4
138402024-02-27T21:49:21.459Z15/30 Read block 195 len 20480
138412024-02-27T21:49:21.459Z16/30 Flush
138422024-02-27T21:49:21.469Z17/30 Write block 199 len 24576 data: 7 8 7 7 7 7
138432024-02-27T21:49:21.469Z18/30 Read block 29 len 28672
138442024-02-27T21:49:21.469Z19/30 Read block 69 len 32768
138452024-02-27T21:49:21.469Z20/30 Write block 543 len 32768 data: 7 6 7 8 8 13 13 12
138462024-02-27T21:49:21.469Z21/30 Read block 105 len 40960
138472024-02-27T21:49:21.479Z22/30 Read block 552 len 28672
138482024-02-27T21:49:21.479Z23/30 Write block 322 len 12288 data: 11 10 9
138492024-02-27T21:49:21.479Z24/30 Read block 412 len 4096
138502024-02-27T21:49:21.479Z25/30 Write block 480 len 40960 data: 9 10 12 14 13 13 15 16 14 16
138512024-02-27T21:49:21.479Z26/30 Write block 181 len 20480 data: 17 19 16 15 12
138522024-02-27T21:49:21.479Z27/30 Read block 539 len 40960
138532024-02-27T21:49:21.479Z28/30 Write block 550 len 8192 data: 13 13
138542024-02-27T21:49:21.479Z29/30 Read block 520 len 20480
138552024-02-27T21:49:21.479Z30/30 Write block 562 len 16384 data: 14 13 10 8
138562024-02-27T21:49:21.479Z----------------------------------------------------------------
138572024-02-27T21:49:21.479Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:25
138582024-02-27T21:49:21.479ZStates: Active Active Active
138592024-02-27T21:49:21.479ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138602024-02-27T21:49:21.479Z 6 Acked 1005 Write 9 Done Done Done false
138612024-02-27T21:49:21.479Z 7 Acked 1006 Write 5 Done Done Done false
138622024-02-27T21:49:21.479Z 8 Acked 1007 Read 10 Done Done Done false
138632024-02-27T21:49:21.479Z 9 Acked 1008 Write 6 Sent Done Done false
138642024-02-27T21:49:21.479Z 10 Acked 1009 Write 2 Sent Done Done false
138652024-02-27T21:49:21.479Z 11 Acked 1010 Write 9 Sent Done Done false
138662024-02-27T21:49:21.480Z 12 Acked 1011 Write 2 Sent Done Done false
138672024-02-27T21:49:21.480Z 13 Acked 1012 Read 9 Sent Done Done false
138682024-02-27T21:49:21.480Z 14 Acked 1013 Write 2 Sent Done Done false
138692024-02-27T21:49:21.480Z 15 Acked 1014 Read 5 Sent Done Done false
138702024-02-27T21:49:21.480Z 16 Acked 1015 Flush 0 Sent Done Done false
138712024-02-27T21:49:21.480Z 17 Acked 1016 Write 6 Sent Done Done false
138722024-02-27T21:49:21.480Z 18 Acked 1017 Read 7 Sent Done Done false
138732024-02-27T21:49:21.480Z 19 Acked 1018 Read 8 Sent Done Done false
138742024-02-27T21:49:21.480Z 20 Acked 1019 Write 8 Sent Done Done false
138752024-02-27T21:49:21.480Z 21 Acked 1020 Read 10 Sent Done Done false
138762024-02-27T21:49:21.480Z 22 Acked 1021 Read 7 Sent Done Done false
138772024-02-27T21:49:21.480Z 23 Acked 1022 Write 3 Sent Done Done false
138782024-02-27T21:49:21.480Z 24 Acked 1023 Read 1 Sent Done Done false
138792024-02-27T21:49:21.480Z 25 Acked 1024 Write 10 Sent Done Done false
138802024-02-27T21:49:21.480Z 26 Acked 1025 Write 5 Sent Done Done false
138812024-02-27T21:49:21.480Z 27 Acked 1026 Read 10 Sent Done Done false
138822024-02-27T21:49:21.480Z 28 Acked 1027 Write 2 Sent Done Done false
138832024-02-27T21:49:21.480Z 29 Acked 1028 Read 5 Sent Done Done false
138842024-02-27T21:49:21.480Z 30 Acked 1029 Write 4 Sent Sent Sent false
138852024-02-27T21:49:21.480Z STATES DS:0 DS:1 DS:2 TOTAL
138862024-02-27T21:49:21.480Z New 0 0 0 0
138872024-02-27T21:49:21.480Z Sent 22 1 1 24
138882024-02-27T21:49:21.480Z Done 3 24 24 51
138892024-02-27T21:49:21.480Z Skipped 0 0 0 0
138902024-02-27T21:49:21.480Z Error 0 0 0 0
138912024-02-27T21:49:21.481ZLast Flush: 1004 1015 1015
138922024-02-27T21:49:21.481ZDownstairs last five completed: 1004 1003 1002 1001 1000
138932024-02-27T21:49:21.481ZUpstairs last five completed: 30 29 28 27 26
138942024-02-27T21:49:21.481Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:19.845642215Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1659}
138952024-02-27T21:49:21.481ZWrote out file "/tmp/test_repair_verify.data"
138962024-02-27T21:49:21.487Z
138972024-02-27T21:49:21.487ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138982024-02-27T21:49:21.505ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138992024-02-27T21:49:21.505Z 0 000-019 124 124 124 229 229 229 F F F
139002024-02-27T21:49:21.505Z 1 020-039 126 126 126 235 235 235 F F F
139012024-02-27T21:49:21.505Z 2 040-059 124 124 124 232 232 232 F F F
139022024-02-27T21:49:21.505Z 3 060-079 124 124 124 232 232 232 F F F
139032024-02-27T21:49:21.505Z 4 080-099 124 124 124 229 229 229 F F F
139042024-02-27T21:49:21.511Z 5 100-119 127 127 127 237 237 237 F F F
139052024-02-27T21:49:21.511Z 6 120-139 124 128 128 231 240 240 T F F <---
139062024-02-27T21:49:21.511Z 7 140-159 128 128 128 238 240 240 F F F <---
139072024-02-27T21:49:21.511Z 8 160-179 127 128 128 237 240 240 F F F <---
139082024-02-27T21:49:21.511Z 9 180-199 126 126 126 234 234 234 F T T <---
139092024-02-27T21:49:21.511Z 10 200-219 122 128 128 225 240 240 F T T <---
139102024-02-27T21:49:21.511Z 11 220-239 124 124 124 231 231 231 F F F
139112024-02-27T21:49:21.511Z 12 240-259 125 125 125 233 233 233 F F F
139122024-02-27T21:49:21.511Z 13 260-279 125 125 125 233 233 233 F F F
139132024-02-27T21:49:21.512Z 14 280-299 127 127 127 237 237 237 F F F
139142024-02-27T21:49:21.512Z 15 300-319 127 127 127 237 237 237 F F F
139152024-02-27T21:49:21.512Z 16 320-339 124 124 124 231 231 231 F T T <---
139162024-02-27T21:49:21.512Z 17 340-359 127 127 127 237 237 237 F F F
139172024-02-27T21:49:21.512Z 18 360-379 126 126 126 236 236 236 F F F
139182024-02-27T21:49:21.512Z 19 380-399 128 128 128 238 238 238 F F F
139192024-02-27T21:49:21.512Z 20 400-419 124 128 128 232 240 240 F F F <---
139202024-02-27T21:49:21.512Z 21 420-439 122 122 122 224 224 224 F F F
139212024-02-27T21:49:21.513Z 22 440-459 113 113 113 208 208 208 F F F
139222024-02-27T21:49:21.513Z 23 460-479 124 124 124 231 231 231 F F F
139232024-02-27T21:49:21.513Z 24 480-499 108 108 108 203 203 203 F T T <---
139242024-02-27T21:49:21.513Z 25 500-519 120 128 128 220 240 240 F F F <---
139252024-02-27T21:49:21.513Z 26 520-539 127 127 127 237 237 237 F F F
139262024-02-27T21:49:21.513Z 27 540-559 124 124 124 231 231 231 F T T <---
139272024-02-27T21:49:21.513Z 28 560-579 124 128 128 231 240 240 T T T <---
139282024-02-27T21:49:21.513Z 29 580-599 126 126 126 234 234 234 F F F
139292024-02-27T21:49:21.513ZMax gen: 128, Max flush: 240
139302024-02-27T21:49:21.513ZError: Difference in extent metadata found!
139312024-02-27T21:49:21.513ZA Difference in extent metadata is expected here
139322024-02-27T21:49:21.513ZOn loop 64
139332024-02-27T21:49:21.513Z
139342024-02-27T21:49:21.513ZVerifying data now
139352024-02-27T21:49:26.084ZNo change after verify
139362024-02-27T21:49:26.085ZLoop: 64 Downstairs dump after verify (and repair):
139372024-02-27T21:49:26.100ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139382024-02-27T21:49:26.100Z 0 000-019 124 124 124 229 229 229 F F F
139392024-02-27T21:49:26.100Z 1 020-039 126 126 126 235 235 235 F F F
139402024-02-27T21:49:26.100Z 2 040-059 124 124 124 232 232 232 F F F
139412024-02-27T21:49:26.100Z 3 060-079 124 124 124 232 232 232 F F F
139422024-02-27T21:49:26.100Z 4 080-099 124 124 124 229 229 229 F F F
139432024-02-27T21:49:26.100Z 5 100-119 127 127 127 237 237 237 F F F
139442024-02-27T21:49:26.103Z 6 120-139 128 128 128 240 240 240 F F F
139452024-02-27T21:49:26.103Z 7 140-159 128 128 128 240 240 240 F F F
139462024-02-27T21:49:26.103Z 8 160-179 128 128 128 240 240 240 F F F
139472024-02-27T21:49:26.103Z 9 180-199 129 129 129 241 241 241 F F F
139482024-02-27T21:49:26.103Z 10 200-219 129 129 129 241 241 241 F F F
139492024-02-27T21:49:26.103Z 11 220-239 124 124 124 231 231 231 F F F
139502024-02-27T21:49:26.103Z 12 240-259 125 125 125 233 233 233 F F F
139512024-02-27T21:49:26.103Z 13 260-279 125 125 125 233 233 233 F F F
139522024-02-27T21:49:26.103Z 14 280-299 127 127 127 237 237 237 F F F
139532024-02-27T21:49:26.103Z 15 300-319 127 127 127 237 237 237 F F F
139542024-02-27T21:49:26.103Z 16 320-339 129 129 129 241 241 241 F F F
139552024-02-27T21:49:26.104Z 17 340-359 127 127 127 237 237 237 F F F
139562024-02-27T21:49:26.104Z 18 360-379 126 126 126 236 236 236 F F F
139572024-02-27T21:49:26.104Z 19 380-399 128 128 128 238 238 238 F F F
139582024-02-27T21:49:26.104Z 20 400-419 128 128 128 240 240 240 F F F
139592024-02-27T21:49:26.104Z 21 420-439 122 122 122 224 224 224 F F F
139602024-02-27T21:49:26.104Z 22 440-459 113 113 113 208 208 208 F F F
139612024-02-27T21:49:26.104Z 23 460-479 124 124 124 231 231 231 F F F
139622024-02-27T21:49:26.104Z 24 480-499 129 129 129 241 241 241 F F F
139632024-02-27T21:49:26.104Z 25 500-519 128 128 128 240 240 240 F F F
139642024-02-27T21:49:26.104Z 26 520-539 127 127 127 237 237 237 F F F
139652024-02-27T21:49:26.104Z 27 540-559 129 129 129 241 241 241 F F F
139662024-02-27T21:49:26.104Z 28 560-579 129 129 129 241 241 241 F F F
139672024-02-27T21:49:26.104Z 29 580-599 126 126 126 234 234 234 F F F
139682024-02-27T21:49:26.104ZMax gen: 129, Max flush: 241
139692024-02-27T21:49:26.104Z
139702024-02-27T21:49:26.104ZBegin loop 65
139712024-02-27T21:49:26.104ZDownstairs to restart: 2
139722024-02-27T21:49:26.121Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.494439034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667}
139732024-02-27T21:49:26.121Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.49472149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667}
139742024-02-27T21:49:26.121Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.494755916Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667}
139752024-02-27T21:49:26.121ZCrucible runtime is spawned
139762024-02-27T21:49:26.127Z{"msg":"Crucible fb6c5d01-b74f-457a-a935-5909c3589b68 has session id: 3b420c29-1909-4b0a-8660-1f7065d78474","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.49480813Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139772024-02-27T21:49:26.127Z{"msg":"Upstairs opts: Upstairs UUID: fb6c5d01-b74f-457a-a935-5909c3589b68, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.494851795Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139782024-02-27T21:49:26.127Z{"msg":"Crucible stats registered with UUID: fb6c5d01-b74f-457a-a935-5909c3589b68","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.494899569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139792024-02-27T21:49:26.127Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.494918197Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667}
139802024-02-27T21:49:26.128Z{"msg":"fb6c5d01-b74f-457a-a935-5909c3589b68 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495083567Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139812024-02-27T21:49:26.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.49514418Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139822024-02-27T21:49:26.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495206883Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139832024-02-27T21:49:26.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495251318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139842024-02-27T21:49:26.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495277595Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139852024-02-27T21:49:26.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495308211Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139862024-02-27T21:49:26.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495339697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139872024-02-27T21:49:26.128Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495474092Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139882024-02-27T21:49:26.128Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495531325Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139892024-02-27T21:49:26.128Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495561241Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139902024-02-27T21:49:26.128Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495607406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139912024-02-27T21:49:26.128Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.495633013Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139922024-02-27T21:49:26.128Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.49565624Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139932024-02-27T21:49:26.128Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.496135234Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139942024-02-27T21:49:26.128Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.49616387Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139952024-02-27T21:49:26.128Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.496179189Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139962024-02-27T21:49:26.128Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.496224473Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139972024-02-27T21:49:26.128Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.496277337Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139982024-02-27T21:49:26.128Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.496307813Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
139992024-02-27T21:49:26.128Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.496737013Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140002024-02-27T21:49:26.128Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.496822363Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140012024-02-27T21:49:26.128Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.496999312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140022024-02-27T21:49:26.129Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497238784Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140032024-02-27T21:49:26.129Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497275079Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140042024-02-27T21:49:26.129Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497290647Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140052024-02-27T21:49:26.129Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497314925Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140062024-02-27T21:49:26.129Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497336992Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140072024-02-27T21:49:26.129Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.49735098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140082024-02-27T21:49:26.129Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497470806Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140092024-02-27T21:49:26.129Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497511521Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140102024-02-27T21:49:26.129Z{"msg":"[0]R flush_numbers[0..12]: [229, 235, 232, 232, 229, 237, 240, 240, 240, 241, 241, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497529689Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140112024-02-27T21:49:26.129Z{"msg":"[0]R generation[0..12]: [124, 126, 124, 124, 124, 127, 128, 128, 128, 129, 129, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497545068Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140122024-02-27T21:49:26.129Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497560726Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140132024-02-27T21:49:26.129Z{"msg":"[1]R flush_numbers[0..12]: [229, 235, 232, 232, 229, 237, 240, 240, 240, 241, 241, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497575094Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140142024-02-27T21:49:26.129Z{"msg":"[1]R generation[0..12]: [124, 126, 124, 124, 124, 127, 128, 128, 128, 129, 129, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497597221Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140152024-02-27T21:49:26.129Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.49761163Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140162024-02-27T21:49:26.129Z{"msg":"[2]R flush_numbers[0..12]: [229, 235, 232, 232, 229, 237, 240, 240, 240, 241, 241, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497625768Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140172024-02-27T21:49:26.129Z{"msg":"[2]R generation[0..12]: [124, 126, 124, 124, 124, 127, 128, 128, 128, 129, 129, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497661424Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140182024-02-27T21:49:26.129Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497684641Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO
140192024-02-27T21:49:26.129Z:"3b420c29-1909-4b0a-8660-1f7065d78474"}
140202024-02-27T21:49:26.129Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497720277Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140212024-02-27T21:49:26.129Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497754243Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140222024-02-27T21:49:26.129Z{"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497770021Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140232024-02-27T21:49:26.129Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497790559Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"":"downstairs","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140242024-02-27T21:49:26.129Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497804247Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140252024-02-27T21:49:26.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497816836Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"0","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140262024-02-27T21:49:26.130Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497850372Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"1","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140272024-02-27T21:49:26.130Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497878628Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"client":"2","":"downstairs","session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140282024-02-27T21:49:26.130Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497895656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140292024-02-27T21:49:26.130Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497908995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140302024-02-27T21:49:26.130Z{"msg":"fb6c5d01-b74f-457a-a935-5909c3589b68 is now active with session: 3b420c29-1909-4b0a-8660-1f7065d78474","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497929682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140312024-02-27T21:49:26.130Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497964148Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667,"session_id":"3b420c29-1909-4b0a-8660-1f7065d78474"}
140322024-02-27T21:49:26.130Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:24.497986896Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667}
140332024-02-27T21:49:26.130ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140342024-02-27T21:49:26.135ZLoading write count information from file "/tmp/test_repair_verify.data"
140352024-02-27T21:49:26.135ZRun Repair workload
140362024-02-27T21:49:26.135Z01/30 Write block 118 len 28672 data: 9 10 12 14 12 13 15
140372024-02-27T21:49:26.135Z02/30 Write block 494 len 32768 data: 9 6 6 6 8 8 6 5
140382024-02-27T21:49:26.135Z03/30 Read block 262 len 40960
140392024-02-27T21:49:26.135Z04/30 Write block 309 len 24576 data: 13 14 14 13 11 12
140402024-02-27T21:49:26.135Z05/30 Flush
140412024-02-27T21:49:26.135Z06/30 Write block 348 len 40960 data: 13 14 13 12 10 9 9 13 12 7
140422024-02-27T21:49:26.135Z07/30 Write block 419 len 4096 data: 12
140432024-02-27T21:49:26.135Z08/30 Read block 408 len 12288
140442024-02-27T21:49:26.138Z09/30 Write block 12 len 4096 data: 15
140452024-02-27T21:49:26.138Z10/30 Write block 355 len 28672 data: 14 13 8 8 9 9 6
140462024-02-27T21:49:26.139Z11/30 Read block 146 len 20480
140472024-02-27T21:49:26.139Z12/30 Read block 101 len 36864
140482024-02-27T21:49:26.142Z13/30 Read block 186 len 16384
140492024-02-27T21:49:26.142Z14/30 Write block 381 len 28672 data: 7 10 10 11 11 10 12
140502024-02-27T21:49:26.142Z15/30 Write block 194 len 8192 data: 9 8
140512024-02-27T21:49:26.142Z16/30 Write block 472 len 16384 data: 7 8 7 9
140522024-02-27T21:49:26.145Z17/30 Read block 262 len 40960
140532024-02-27T21:49:26.145Z18/30 Read block 109 len 32768
140542024-02-27T21:49:26.145Z19/30 Read block 270 len 32768
140552024-02-27T21:49:26.145Z20/30 Write block 180 len 8192 data: 15 18
140562024-02-27T21:49:26.157Z21/30 Read block 289 len 36864
140572024-02-27T21:49:26.157Z22/30 Read block 536 len 36864
140582024-02-27T21:49:26.158Z23/30 Flush
140592024-02-27T21:49:26.172Z24/30 Write block 67 len 8192 data: 8 8
140602024-02-27T21:49:26.173Z25/30 Flush
140612024-02-27T21:49:26.173Z26/30 Write block 587 len 40960 data: 11 11 12 12 9 10 10 8 7 8
140622024-02-27T21:49:26.173Z27/30 Read block 141 len 28672
140632024-02-27T21:49:26.173Z28/30 Read block 464 len 24576
140642024-02-27T21:49:26.173Z29/30 Read block 261 len 8192
140652024-02-27T21:49:26.173Z30/30 Read block 119 len 8192
140662024-02-27T21:49:26.173Z----------------------------------------------------------------
140672024-02-27T21:49:26.173Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:25
140682024-02-27T21:49:26.173ZStates: Active Active Active
140692024-02-27T21:49:26.173ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140702024-02-27T21:49:26.173Z 6 Acked 1005 Write 10 Done Done Done false
140712024-02-27T21:49:26.173Z 7 Acked 1006 Write 1 Done Done Done false
140722024-02-27T21:49:26.173Z 8 Acked 1007 Read 3 Done Done Done false
140732024-02-27T21:49:26.173Z 9 Acked 1008 Write 1 Done Done Done false
140742024-02-27T21:49:26.173Z 10 Acked 1009 Write 7 Done Done Done false
140752024-02-27T21:49:26.173Z 11 Acked 1010 Read 5 Done Done Done false
140762024-02-27T21:49:26.173Z 12 Acked 1011 Read 9 Done Done Sent false
140772024-02-27T21:49:26.173Z 13 Acked 1012 Read 4 Done Done Sent false
140782024-02-27T21:49:26.173Z 14 Acked 1013 Write 7 Done Done Sent false
140792024-02-27T21:49:26.174Z 15 Acked 1014 Write 2 Done Done Sent false
140802024-02-27T21:49:26.174Z 16 Acked 1015 Write 4 Done Done Sent false
140812024-02-27T21:49:26.174Z 17 Acked 1016 Read 10 Done Done Sent false
140822024-02-27T21:49:26.174Z 18 Acked 1017 Read 8 Done Done Sent false
140832024-02-27T21:49:26.174Z 19 Acked 1018 Read 8 Done Done Sent false
140842024-02-27T21:49:26.174Z 20 Acked 1019 Write 2 Done Done Sent false
140852024-02-27T21:49:26.174Z 21 Acked 1020 Read 9 Done Done Sent false
140862024-02-27T21:49:26.174Z 22 Acked 1021 Read 9 Done Done Sent false
140872024-02-27T21:49:26.174Z 23 Acked 1022 Flush 0 Done Done Sent false
140882024-02-27T21:49:26.174Z 24 Acked 1023 Write 2 Done Done Sent false
140892024-02-27T21:49:26.174Z 25 Acked 1024 Flush 0 Done Done Sent false
140902024-02-27T21:49:26.174Z 26 Acked 1025 Write 10 Done Done Sent false
140912024-02-27T21:49:26.174Z 27 Acked 1026 Read 7 Done Done Sent false
140922024-02-27T21:49:26.174Z 28 Acked 1027 Read 6 Done Done Sent false
140932024-02-27T21:49:26.174Z 29 Acked 1028 Read 2 Done Done Sent false
140942024-02-27T21:49:26.174Z 30 Acked 1029 Read 2 Done Done Sent false
140952024-02-27T21:49:26.174Z STATES DS:0 DS:1 DS:2 TOTAL
140962024-02-27T21:49:26.174Z New 0 0 0 0
140972024-02-27T21:49:26.174Z Sent 0 0 19 19
140982024-02-27T21:49:26.174Z Done 25 25 6 56
140992024-02-27T21:49:26.174Z Skipped 0 0 0 0
141002024-02-27T21:49:26.174Z Error 0 0 0 0
141012024-02-27T21:49:26.174ZLast Flush: 1024 1024 1004
141022024-02-27T21:49:26.174ZDownstairs last five completed: 1004 1003 1002 1001 1000
141032024-02-27T21:49:26.175ZUpstairs last five completed: 30 29 28 27 26
141042024-02-27T21:49:26.175Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:24.532067771Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1667}
141052024-02-27T21:49:26.175ZWrote out file "/tmp/test_repair_verify.data"
141062024-02-27T21:49:26.175Z
141072024-02-27T21:49:26.175ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141082024-02-27T21:49:26.183ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141092024-02-27T21:49:26.183Z 0 000-019 130 130 124 243 243 229 F F T <---
141102024-02-27T21:49:26.184Z 1 020-039 126 126 126 235 235 235 F F F
141112024-02-27T21:49:26.184Z 2 040-059 124 124 124 232 232 232 F F F
141122024-02-27T21:49:26.184Z 3 060-079 130 130 124 244 244 232 F F F <---
141132024-02-27T21:49:26.184Z 4 080-099 124 124 124 229 229 229 F F F
141142024-02-27T21:49:26.184Z 5 100-119 130 130 130 242 242 242 F F F
141152024-02-27T21:49:26.184Z 6 120-139 130 130 130 242 242 242 F F F
141162024-02-27T21:49:26.187Z 7 140-159 128 128 128 240 240 240 F F F
141172024-02-27T21:49:26.187Z 8 160-179 128 128 128 240 240 240 F F F
141182024-02-27T21:49:26.187Z 9 180-199 130 130 129 243 243 241 F F F <---
141192024-02-27T21:49:26.187Z 10 200-219 129 129 129 241 241 241 F F F
141202024-02-27T21:49:26.187Z 11 220-239 124 124 124 231 231 231 F F F
141212024-02-27T21:49:26.187Z 12 240-259 125 125 125 233 233 233 F F F
141222024-02-27T21:49:26.187Z 13 260-279 125 125 125 233 233 233 F F F
141232024-02-27T21:49:26.187Z 14 280-299 127 127 127 237 237 237 F F F
141242024-02-27T21:49:26.188Z 15 300-319 130 130 130 242 242 242 F F F
141252024-02-27T21:49:26.188Z 16 320-339 129 129 129 241 241 241 F F F
141262024-02-27T21:49:26.188Z 17 340-359 130 130 127 243 243 237 F F T <---
141272024-02-27T21:49:26.188Z 18 360-379 130 130 126 243 243 236 F F T <---
141282024-02-27T21:49:26.188Z 19 380-399 130 130 128 243 243 238 F F F <---
141292024-02-27T21:49:26.188Z 20 400-419 130 130 128 243 243 240 F F T <---
141302024-02-27T21:49:26.188Z 21 420-439 122 122 122 224 224 224 F F F
141312024-02-27T21:49:26.188Z 22 440-459 113 113 113 208 208 208 F F F
141322024-02-27T21:49:26.188Z 23 460-479 130 130 124 243 243 231 F F F <---
141332024-02-27T21:49:26.188Z 24 480-499 130 130 130 242 242 242 F F F
141342024-02-27T21:49:26.188Z 25 500-519 130 130 130 242 242 242 F F F
141352024-02-27T21:49:26.188Z 26 520-539 127 127 127 237 237 237 F F F
141362024-02-27T21:49:26.188Z 27 540-559 129 129 129 241 241 241 F F F
141372024-02-27T21:49:26.188Z 28 560-579 129 129 129 241 241 241 F F F
141382024-02-27T21:49:26.188Z 29 580-599 126 126 126 234 234 234 T T F <---
141392024-02-27T21:49:26.188ZMax gen: 130, Max flush: 244
141402024-02-27T21:49:26.188ZError: Difference in extent metadata found!
141412024-02-27T21:49:26.188ZA Difference in extent metadata is expected here
141422024-02-27T21:49:26.188ZOn loop 65
141432024-02-27T21:49:26.188Z
141442024-02-27T21:49:26.188ZVerifying data now
141452024-02-27T21:49:31.668ZNo change after verify
141462024-02-27T21:49:31.668ZLoop: 65 Downstairs dump after verify (and repair):
141472024-02-27T21:49:31.685ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141482024-02-27T21:49:31.685Z 0 000-019 130 130 130 243 243 243 F F F
141492024-02-27T21:49:31.685Z 1 020-039 126 126 126 235 235 235 F F F
141502024-02-27T21:49:31.686Z 2 040-059 124 124 124 232 232 232 F F F
141512024-02-27T21:49:31.686Z 3 060-079 130 130 130 244 244 244 F F F
141522024-02-27T21:49:31.686Z 4 080-099 124 124 124 229 229 229 F F F
141532024-02-27T21:49:31.686Z 5 100-119 130 130 130 242 242 242 F F F
141542024-02-27T21:49:31.686Z 6 120-139 130 130 130 242 242 242 F F F
141552024-02-27T21:49:31.686Z 7 140-159 128 128 128 240 240 240 F F F
141562024-02-27T21:49:31.686Z 8 160-179 128 128 128 240 240 240 F F F
141572024-02-27T21:49:31.686Z 9 180-199 130 130 130 243 243 243 F F F
141582024-02-27T21:49:31.686Z 10 200-219 129 129 129 241 241 241 F F F
141592024-02-27T21:49:31.686Z 11 220-239 124 124 124 231 231 231 F F F
141602024-02-27T21:49:31.686Z 12 240-259 125 125 125 233 233 233 F F F
141612024-02-27T21:49:31.686Z 13 260-279 125 125 125 233 233 233 F F F
141622024-02-27T21:49:31.686Z 14 280-299 127 127 127 237 237 237 F F F
141632024-02-27T21:49:31.686Z 15 300-319 130 130 130 242 242 242 F F F
141642024-02-27T21:49:31.686Z 16 320-339 129 129 129 241 241 241 F F F
141652024-02-27T21:49:31.686Z 17 340-359 130 130 130 243 243 243 F F F
141662024-02-27T21:49:31.686Z 18 360-379 130 130 130 243 243 243 F F F
141672024-02-27T21:49:31.686Z 19 380-399 130 130 130 243 243 243 F F F
141682024-02-27T21:49:31.686Z 20 400-419 130 130 130 243 243 243 F F F
141692024-02-27T21:49:31.696Z 21 420-439 122 122 122 224 224 224 F F F
141702024-02-27T21:49:31.696Z 22 440-459 113 113 113 208 208 208 F F F
141712024-02-27T21:49:31.696Z 23 460-479 130 130 130 243 243 243 F F F
141722024-02-27T21:49:31.696Z 24 480-499 130 130 130 242 242 242 F F F
141732024-02-27T21:49:31.696Z 25 500-519 130 130 130 242 242 242 F F F
141742024-02-27T21:49:31.696Z 26 520-539 127 127 127 237 237 237 F F F
141752024-02-27T21:49:31.696Z 27 540-559 129 129 129 241 241 241 F F F
141762024-02-27T21:49:31.696Z 28 560-579 129 129 129 241 241 241 F F F
141772024-02-27T21:49:31.696Z 29 580-599 131 131 131 245 245 245 F F F
141782024-02-27T21:49:31.696ZMax gen: 131, Max flush: 245
141792024-02-27T21:49:31.696Z
141802024-02-27T21:49:31.696ZBegin loop 66
141812024-02-27T21:49:31.696ZDownstairs to restart: 2
141822024-02-27T21:49:31.708Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.080727941Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675}
141832024-02-27T21:49:31.708Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.080965813Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675}
141842024-02-27T21:49:31.708Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.080983691Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675}
141852024-02-27T21:49:31.708Z{"msg":"Crucible 2d46814c-f96e-4ae8-8609-0a6d66721218 has session id: c0f304ff-3498-48b6-a4b2-ac1551ea725d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.0809972Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141862024-02-27T21:49:31.708Z{"msg":"Upstairs opts: Upstairs UUID: 2d46814c-f96e-4ae8-8609-0a6d66721218, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081027036Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141872024-02-27T21:49:31.723Z{"msg":"Crucible stats registered with UUID: 2d46814c-f96e-4ae8-8609-0a6d66721218","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081041944Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141882024-02-27T21:49:31.723ZCrucible runtime is spawned
141892024-02-27T21:49:31.723Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081175259Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675}
141902024-02-27T21:49:31.723Z{"msg":"2d46814c-f96e-4ae8-8609-0a6d66721218 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081265958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141912024-02-27T21:49:31.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081308243Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141922024-02-27T21:49:31.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081338039Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141932024-02-27T21:49:31.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081356627Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141942024-02-27T21:49:31.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081398722Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141952024-02-27T21:49:31.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.08142081Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141962024-02-27T21:49:31.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081456385Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141972024-02-27T21:49:31.723Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081473213Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141982024-02-27T21:49:31.724Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081513569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
141992024-02-27T21:49:31.724Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081540765Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142002024-02-27T21:49:31.724Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081661141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142012024-02-27T21:49:31.724Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081697237Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142022024-02-27T21:49:31.724Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.081788996Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142032024-02-27T21:49:31.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.082452228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142042024-02-27T21:49:31.724Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.082534598Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142052024-02-27T21:49:31.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.08260438Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142062024-02-27T21:49:31.724Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.082667563Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142072024-02-27T21:49:31.724Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083044938Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142082024-02-27T21:49:31.724Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083169034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142092024-02-27T21:49:31.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083275921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142102024-02-27T21:49:31.724Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083306397Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142112024-02-27T21:49:31.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083471238Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142122024-02-27T21:49:31.724Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083518992Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142132024-02-27T21:49:31.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083566447Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142142024-02-27T21:49:31.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083671905Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142152024-02-27T21:49:31.724Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.08370602Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142162024-02-27T21:49:31.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083721759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142172024-02-27T21:49:31.724Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.083942862Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142182024-02-27T21:49:31.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.08439002Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142192024-02-27T21:49:31.724Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084434165Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142202024-02-27T21:49:31.724Z{"msg":"[0]R flush_numbers[0..12]: [243, 235, 232, 244, 229, 242, 242, 240, 240, 243, 241, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084452842Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142212024-02-27T21:49:31.725Z{"msg":"[0]R generation[0..12]: [130, 126, 124, 130, 124, 130, 130, 128, 128, 130, 129, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.0844756Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142222024-02-27T21:49:31.725Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084491628Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142232024-02-27T21:49:31.725Z{"msg":"[1]R flush_numbers[0..12]: [243, 235, 232, 244, 229, 242, 242, 240, 240, 243, 241, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084512975Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142242024-02-27T21:49:31.725Z{"msg":"[1]R generation[0..12]: [130, 126, 124, 130, 124, 130, 130, 128, 128, 130, 129, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084527234Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142252024-02-27T21:49:31.725Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084548941Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142262024-02-27T21:49:31.725ZWait for a query_work_queue command to finish before sending IO
142272024-02-27T21:49:31.725Z{"msg":"[2]R flush_numbers[0..12]: [243, 235, 232, 244, 229, 242, 242, 240, 240, 243, 241, 231]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084572758Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142282024-02-27T21:49:31.725Z{"msg":"[2]R generation[0..12]: [130, 126, 124, 130, 124, 130, 130, 128, 128, 130, 129, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084587177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142292024-02-27T21:49:31.725Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084613614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142302024-02-27T21:49:31.725Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084627392Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142312024-02-27T21:49:31.725Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.08464074Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142322024-02-27T21:49:31.725Z{"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084665957Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142332024-02-27T21:49:31.725Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084680046Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"":"downstairs","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142342024-02-27T21:49:31.725Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084694174Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142352024-02-27T21:49:31.725Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084713302Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"0","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142362024-02-27T21:49:31.725Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.0847271Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"1","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142372024-02-27T21:49:31.725Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084740828Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"client":"2","":"downstairs","session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142382024-02-27T21:49:31.725Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084754427Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142392024-02-27T21:49:31.725Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084773645Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142402024-02-27T21:49:31.725Z{"msg":"2d46814c-f96e-4ae8-8609-0a6d66721218 is now active with session: c0f304ff-3498-48b6-a4b2-ac1551ea725d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084786963Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142412024-02-27T21:49:31.725Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084805861Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675,"session_id":"c0f304ff-3498-48b6-a4b2-ac1551ea725d"}
142422024-02-27T21:49:31.725Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:30.084818589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675}
142432024-02-27T21:49:31.725ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142442024-02-27T21:49:31.729ZLoading write count information from file "/tmp/test_repair_verify.data"
142452024-02-27T21:49:31.730ZRun Repair workload
142462024-02-27T21:49:31.730Z01/30 Write block 276 len 28672 data: 9 8 7 6 5 4 5
142472024-02-27T21:49:31.730Z02/30 Write block 293 len 20480 data: 12 11 12 12 12
142482024-02-27T21:49:31.730Z03/30 Write block 423 len 8192 data: 7 8
142492024-02-27T21:49:31.730Z04/30 Write block 467 len 8192 data: 10 10
142502024-02-27T21:49:31.730Z05/30 Write block 294 len 32768 data: 12 13 13 13 12 9 10 10
142512024-02-27T21:49:31.730Z06/30 Write block 507 len 32768 data: 7 5 9 11 16 15 15 13
142522024-02-27T21:49:31.730Z07/30 Read block 108 len 32768
142532024-02-27T21:49:31.730Z08/30 Flush
142542024-02-27T21:49:31.730Z09/30 Write block 102 len 20480 data: 12 11 9 11 13
142552024-02-27T21:49:31.730Z10/30 Read block 236 len 16384
142562024-02-27T21:49:31.730Z11/30 Read block 96 len 24576
142572024-02-27T21:49:31.730Z12/30 Write block 406 len 28672 data: 7 5 9 9 8 7 9
142582024-02-27T21:49:31.730Z13/30 Read block 427 len 28672
142592024-02-27T21:49:31.730Z14/30 Read block 266 len 4096
142602024-02-27T21:49:31.730Z15/30 Write block 476 len 16384 data: 11 9 9 8
142612024-02-27T21:49:31.730Z16/30 Read block 217 len 4096
142622024-02-27T21:49:31.730Z17/30 Read block 9 len 20480
142632024-02-27T21:49:31.733Z18/30 Read block 162 len 20480
142642024-02-27T21:49:31.733Z19/30 Write block 239 len 32768 data: 10 6 7 6 7 10 11 12
142652024-02-27T21:49:31.733Z20/30 Flush
142662024-02-27T21:49:31.736Z21/30 Write block 15 len 28672 data: 14 14 13 12 11 11 9
142672024-02-27T21:49:31.736Z22/30 Flush
142682024-02-27T21:49:31.739Z23/30 Write block 342 len 36864 data: 12 12 12 10 10 10 14 15 14
142692024-02-27T21:49:31.739Z24/30 Read block 487 len 4096
142702024-02-27T21:49:31.739Z25/30 Read block 40 len 28672
142712024-02-27T21:49:31.742Z26/30 Read block 386 len 40960
142722024-02-27T21:49:31.742Z27/30 Read block 527 len 8192
142732024-02-27T21:49:31.742Z28/30 Read block 444 len 16384
142742024-02-27T21:49:31.747Z29/30 Read block 42 len 40960
142752024-02-27T21:49:31.747Z30/30 Write block 256 len 40960 data: 14 10 13 13 12 10 10 9 10 10
142762024-02-27T21:49:31.747Z----------------------------------------------------------------
142772024-02-27T21:49:31.747Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
142782024-02-27T21:49:31.747ZStates: Active Active Active
142792024-02-27T21:49:31.747ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142802024-02-27T21:49:31.747Z 1 Acked 1000 Write 7 Done Done Sent false
142812024-02-27T21:49:31.748Z 2 Acked 1001 Write 5 Done Done Sent false
142822024-02-27T21:49:31.748Z 3 Acked 1002 Write 2 Done Done Sent false
142832024-02-27T21:49:31.748Z 4 Acked 1003 Write 2 Done Done Sent false
142842024-02-27T21:49:31.748Z 5 Acked 1004 Write 8 Done Done Sent false
142852024-02-27T21:49:31.748Z 6 Acked 1005 Write 8 Done Done Sent false
142862024-02-27T21:49:31.748Z 7 Acked 1006 Read 8 Done Done Sent false
142872024-02-27T21:49:31.748Z 8 Acked 1007 Flush 0 Done Done Sent false
142882024-02-27T21:49:31.748Z 9 Acked 1008 Write 5 Done Done Sent false
142892024-02-27T21:49:31.748Z 10 Acked 1009 Read 4 Done Done Sent false
142902024-02-27T21:49:31.748Z 11 Acked 1010 Read 6 Done Done Sent false
142912024-02-27T21:49:31.748Z 12 Acked 1011 Write 7 Done Done Sent false
142922024-02-27T21:49:31.748Z 13 Acked 1012 Read 7 Done Done Sent false
142932024-02-27T21:49:31.748Z 14 Acked 1013 Read 1 Done Done Sent false
142942024-02-27T21:49:31.748Z 15 Acked 1014 Write 4 Done Done Sent false
142952024-02-27T21:49:31.748Z 16 Acked 1015 Read 1 Done Done Sent false
142962024-02-27T21:49:31.748Z 17 Acked 1016 Read 5 Done Done Sent false
142972024-02-27T21:49:31.748Z 18 Acked 1017 Read 5 Done Done Sent false
142982024-02-27T21:49:31.748Z 19 Acked 1018 Write 8 Done Done Sent false
142992024-02-27T21:49:31.748Z 20 Acked 1019 Flush 0 Done Done Sent false
143002024-02-27T21:49:31.748Z 21 Acked 1020 Write 7 Done Done Sent false
143012024-02-27T21:49:31.748Z 22 Acked 1021 Flush 0 Done Done Sent false
143022024-02-27T21:49:31.748Z 23 Acked 1022 Write 9 Done Done Sent false
143032024-02-27T21:49:31.748Z 24 Acked 1023 Read 1 Done Done Sent false
143042024-02-27T21:49:31.748Z 25 Acked 1024 Read 7 Done Done Sent false
143052024-02-27T21:49:31.749Z 26 Acked 1025 Read 10 Done Done Sent false
143062024-02-27T21:49:31.749Z 27 Acked 1026 Read 2 Done Done Sent false
143072024-02-27T21:49:31.749Z 28 Acked 1027 Read 4 Done Done Sent false
143082024-02-27T21:49:31.749Z 29 Acked 1028 Read 10 Done Done Sent false
143092024-02-27T21:49:31.749Z 30 Acked 1029 Write 10 Sent Sent Sent false
143102024-02-27T21:49:31.749Z STATES DS:0 DS:1 DS:2 TOTAL
143112024-02-27T21:49:31.749Z New 0 0 0 0
143122024-02-27T21:49:31.749Z Sent 1 1 30 32
143132024-02-27T21:49:31.749Z Done 29 29 0 58
143142024-02-27T21:49:31.749Z Skipped 0 0 0 0
143152024-02-27T21:49:31.749Z Error 0 0 0 0
143162024-02-27T21:49:31.749ZLast Flush: 1021 1021 0
143172024-02-27T21:49:31.749ZDownstairs last five completed:
143182024-02-27T21:49:31.749ZUpstairs last five completed: 30 29 28 27 26
143192024-02-27T21:49:31.749Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:30.118800556Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1675}
143202024-02-27T21:49:31.749ZWrote out file "/tmp/test_repair_verify.data"
143212024-02-27T21:49:31.753Z
143222024-02-27T21:49:31.755ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143232024-02-27T21:49:31.770ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143242024-02-27T21:49:31.770Z 0 000-019 132 132 130 248 248 243 F F F <---
143252024-02-27T21:49:31.771Z 1 020-039 132 132 126 248 248 235 F F F <---
143262024-02-27T21:49:31.771Z 2 040-059 124 124 124 232 232 232 F F F
143272024-02-27T21:49:31.771Z 3 060-079 130 130 130 244 244 244 F F F
143282024-02-27T21:49:31.771Z 4 080-099 124 124 124 229 229 229 F F F
143292024-02-27T21:49:31.771Z 5 100-119 132 132 130 247 247 242 F F F <---
143302024-02-27T21:49:31.771Z 6 120-139 130 130 130 242 242 242 F F F
143312024-02-27T21:49:31.771Z 7 140-159 128 128 128 240 240 240 F F F
143322024-02-27T21:49:31.771Z 8 160-179 128 128 128 240 240 240 F F F
143332024-02-27T21:49:31.771Z 9 180-199 130 130 130 243 243 243 F F F
143342024-02-27T21:49:31.771Z 10 200-219 129 129 129 241 241 241 F F F
143352024-02-27T21:49:31.771Z 11 220-239 132 132 124 247 247 231 F F F <---
143362024-02-27T21:49:31.771Z 12 240-259 132 132 125 247 247 233 T T F <---
143372024-02-27T21:49:31.771Z 13 260-279 132 132 125 246 246 233 T T F <---
143382024-02-27T21:49:31.771Z 14 280-299 132 132 127 246 246 237 F F F <---
143392024-02-27T21:49:31.771Z 15 300-319 132 132 130 246 246 242 F F F <---
143402024-02-27T21:49:31.771Z 16 320-339 129 129 129 241 241 241 F F F
143412024-02-27T21:49:31.771Z 17 340-359 130 130 130 243 243 243 T T F <---
143422024-02-27T21:49:31.771Z 18 360-379 130 130 130 243 243 243 F F F
143432024-02-27T21:49:31.771Z 19 380-399 130 130 130 243 243 243 F F F
143442024-02-27T21:49:31.775Z 20 400-419 132 132 130 247 247 243 F F F <---
143452024-02-27T21:49:31.775Z 21 420-439 132 132 122 246 246 224 F F F <---
143462024-02-27T21:49:31.775Z 22 440-459 113 113 113 208 208 208 F F F
143472024-02-27T21:49:31.775Z 23 460-479 132 132 130 247 247 243 F F F <---
143482024-02-27T21:49:31.775Z 24 480-499 130 130 130 242 242 242 F F F
143492024-02-27T21:49:31.775Z 25 500-519 132 132 130 246 246 242 F F F <---
143502024-02-27T21:49:31.775Z 26 520-539 127 127 127 237 237 237 F F F
143512024-02-27T21:49:31.775Z 27 540-559 129 129 129 241 241 241 F F F
143522024-02-27T21:49:31.775Z 28 560-579 129 129 129 241 241 241 F F F
143532024-02-27T21:49:31.775Z 29 580-599 131 131 131 245 245 245 F F F
143542024-02-27T21:49:31.775ZMax gen: 132, Max flush: 248
143552024-02-27T21:49:31.775ZError: Difference in extent metadata found!
143562024-02-27T21:49:31.775ZA Difference in extent metadata is expected here
143572024-02-27T21:49:31.775ZOn loop 66
143582024-02-27T21:49:31.775Z
143592024-02-27T21:49:31.782ZVerifying data now
143602024-02-27T21:49:36.438ZNo change after verify
143612024-02-27T21:49:36.438ZLoop: 66 Downstairs dump after verify (and repair):
143622024-02-27T21:49:36.452ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143632024-02-27T21:49:36.452Z 0 000-019 132 132 132 248 248 248 F F F
143642024-02-27T21:49:36.456Z 1 020-039 132 132 132 248 248 248 F F F
143652024-02-27T21:49:36.456Z 2 040-059 124 124 124 232 232 232 F F F
143662024-02-27T21:49:36.456Z 3 060-079 130 130 130 244 244 244 F F F
143672024-02-27T21:49:36.456Z 4 080-099 124 124 124 229 229 229 F F F
143682024-02-27T21:49:36.456Z 5 100-119 132 132 132 247 247 247 F F F
143692024-02-27T21:49:36.456Z 6 120-139 130 130 130 242 242 242 F F F
143702024-02-27T21:49:36.456Z 7 140-159 128 128 128 240 240 240 F F F
143712024-02-27T21:49:36.456Z 8 160-179 128 128 128 240 240 240 F F F
143722024-02-27T21:49:36.456Z 9 180-199 130 130 130 243 243 243 F F F
143732024-02-27T21:49:36.456Z 10 200-219 129 129 129 241 241 241 F F F
143742024-02-27T21:49:36.456Z 11 220-239 132 132 132 247 247 247 F F F
143752024-02-27T21:49:36.456Z 12 240-259 133 133 133 249 249 249 F F F
143762024-02-27T21:49:36.456Z 13 260-279 133 133 133 249 249 249 F F F
143772024-02-27T21:49:36.456Z 14 280-299 132 132 132 246 246 246 F F F
143782024-02-27T21:49:36.456Z 15 300-319 132 132 132 246 246 246 F F F
143792024-02-27T21:49:36.456Z 16 320-339 129 129 129 241 241 241 F F F
143802024-02-27T21:49:36.456Z 17 340-359 133 133 133 249 249 249 F F F
143812024-02-27T21:49:36.456Z 18 360-379 130 130 130 243 243 243 F F F
143822024-02-27T21:49:36.456Z 19 380-399 130 130 130 243 243 243 F F F
143832024-02-27T21:49:36.456Z 20 400-419 132 132 132 247 247 247 F F F
143842024-02-27T21:49:36.457Z 21 420-439 132 132 132 246 246 246 F F F
143852024-02-27T21:49:36.457Z 22 440-459 113 113 113 208 208 208 F F F
143862024-02-27T21:49:36.457Z 23 460-479 132 132 132 247 247 247 F F F
143872024-02-27T21:49:36.457Z 24 480-499 130 130 130 242 242 242 F F F
143882024-02-27T21:49:36.457Z 25 500-519 132 132 132 246 246 246 F F F
143892024-02-27T21:49:36.457Z 26 520-539 127 127 127 237 237 237 F F F
143902024-02-27T21:49:36.457Z 27 540-559 129 129 129 241 241 241 F F F
143912024-02-27T21:49:36.457Z 28 560-579 129 129 129 241 241 241 F F F
143922024-02-27T21:49:36.457Z 29 580-599 131 131 131 245 245 245 F F F
143932024-02-27T21:49:36.457ZMax gen: 133, Max flush: 249
143942024-02-27T21:49:36.457Z
143952024-02-27T21:49:36.457ZBegin loop 67
143962024-02-27T21:49:36.457ZDownstairs to restart: 2
143972024-02-27T21:49:36.476Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.845463761Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683}
143982024-02-27T21:49:36.476Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.845819269Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683}
143992024-02-27T21:49:36.483Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.845853935Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683}
144002024-02-27T21:49:36.483Z{"msg":"Crucible 9be0abdf-9b08-48a6-b84d-f642dbf4f3cb has session id: 78d83c5c-09ad-40a4-bea1-3f3ffca8e145","v":0,"name":"crucible","level":30Crucible runtime is spawned
144012024-02-27T21:49:36.483Z,"time":"2024-02-27T21:49:34.845917067Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144022024-02-27T21:49:36.483Z{"msg":"Upstairs opts: Upstairs UUID: 9be0abdf-9b08-48a6-b84d-f642dbf4f3cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.845969111Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144032024-02-27T21:49:36.483Z{"msg":"Crucible stats registered with UUID: 9be0abdf-9b08-48a6-b84d-f642dbf4f3cb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.845990569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144042024-02-27T21:49:36.483Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846033534Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683}
144052024-02-27T21:49:36.483Z{"msg":"9be0abdf-9b08-48a6-b84d-f642dbf4f3cb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846164558Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144062024-02-27T21:49:36.483Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846196324Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144072024-02-27T21:49:36.483Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846220332Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144082024-02-27T21:49:36.483Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846263016Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144092024-02-27T21:49:36.483Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846300062Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144102024-02-27T21:49:36.483Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846325389Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144112024-02-27T21:49:36.484Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846343987Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144122024-02-27T21:49:36.484Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846361995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144132024-02-27T21:49:36.484Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846438276Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144142024-02-27T21:49:36.484Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846497539Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144152024-02-27T21:49:36.484Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846526685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144162024-02-27T21:49:36.484Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846554162Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144172024-02-27T21:49:36.484Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.846633553Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144182024-02-27T21:49:36.484Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.847196157Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144192024-02-27T21:49:36.484Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.847232862Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144202024-02-27T21:49:36.484Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.84724882Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144212024-02-27T21:49:36.484Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.847270398Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144222024-02-27T21:49:36.484Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.847316312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144232024-02-27T21:49:36.484Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.847346259Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144242024-02-27T21:49:36.484Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.847810044Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144252024-02-27T21:49:36.484Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.847852879Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144262024-02-27T21:49:36.484Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848053336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144272024-02-27T21:49:36.484Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848335123Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144282024-02-27T21:49:36.484Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848387226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144292024-02-27T21:49:36.484Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848466937Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144302024-02-27T21:49:36.484Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848512202Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144312024-02-27T21:49:36.484Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848534409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144322024-02-27T21:49:36.484Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848561506Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144332024-02-27T21:49:36.484Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848579124Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144342024-02-27T21:49:36.484Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848600071Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144352024-02-27T21:49:36.484Z{"msg":"[0]R flush_numbers[0..12]: [248, 248, 232, 244, 229, 247, 242, 240, 240, 243, 241, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.84861545Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144362024-02-27T21:49:36.485Z{"msg":"[0]R generation[0..12]: [132, 132, 124, 130, 124, 132, 130, 128, 128, 130, 129, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848636607Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144372024-02-27T21:49:36.485Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848651135Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144382024-02-27T21:49:36.485Z{"msg":"[1]R flush_numbers[0..12]: [248, 248, 232, 244, 229, 247, 242, 240, 240, 243, 241, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848671943Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144392024-02-27T21:49:36.485Z{"msg":"[1]R generation[0..12]: [132, 132, 124, 130, 124, 132, 130, 128, 128, 130, 129, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848685901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144402024-02-27T21:49:36.485Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
144412024-02-27T21:49:36.485Z,"time":"2024-02-27T21:49:34.848706749Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144422024-02-27T21:49:36.485Z{"msg":"[2]R flush_numbers[0..12]: [248, 248, 232, 244, 229, 247, 242, 240, 240, 243, 241, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848721307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144432024-02-27T21:49:36.485Z{"msg":"[2]R generation[0..12]: [132, 132, 124, 130, 124, 132, 130, 128, 128, 130, 129, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848741345Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144442024-02-27T21:49:36.485Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848755283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144452024-02-27T21:49:36.485Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.84877809Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144462024-02-27T21:49:36.485Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848794728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144472024-02-27T21:49:36.485Z{"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848816356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144482024-02-27T21:49:36.485Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848830684Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"":"downstairs","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144492024-02-27T21:49:36.485Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848844372Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144502024-02-27T21:49:36.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.84886403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"0","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144512024-02-27T21:49:36.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848877628Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"1","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144522024-02-27T21:49:36.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848890837Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"client":"2","":"downstairs","session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144532024-02-27T21:49:36.485Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848912124Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144542024-02-27T21:49:36.485Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848925693Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144552024-02-27T21:49:36.485Z{"msg":"9be0abdf-9b08-48a6-b84d-f642dbf4f3cb is now active with session: 78d83c5c-09ad-40a4-bea1-3f3ffca8e145","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.84894604Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144562024-02-27T21:49:36.485Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848959149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683,"session_id":"78d83c5c-09ad-40a4-bea1-3f3ffca8e145"}
144572024-02-27T21:49:36.485Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:34.848972108Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683}
144582024-02-27T21:49:36.485ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144592024-02-27T21:49:36.486ZLoading write count information from file "/tmp/test_repair_verify.data"
144602024-02-27T21:49:36.486ZRun Repair workload
144612024-02-27T21:49:36.486Z01/30 Write block 417 len 28672 data: 16 14 13 8 6 7 8
144622024-02-27T21:49:36.486Z02/30 Write block 449 len 24576 data: 7 8 9 8 9 7
144632024-02-27T21:49:36.486Z03/30 Flush
144642024-02-27T21:49:36.490Z04/30 Write block 518 len 40960 data: 9 8 11 8 9 7 8 7 8 8
144652024-02-27T21:49:36.490Z05/30 Read block 275 len 32768
144662024-02-27T21:49:36.490Z06/30 Read block 424 len 12288
144672024-02-27T21:49:36.490Z07/30 Read block 556 len 32768
144682024-02-27T21:49:36.490Z08/30 Read block 115 len 32768
144692024-02-27T21:49:36.490Z09/30 Write block 338 len 20480 data: 14 13 15 15 13
144702024-02-27T21:49:36.490Z10/30 Write block 350 len 8192 data: 15 13
144712024-02-27T21:49:36.490Z11/30 Read block 94 len 12288
144722024-02-27T21:49:36.494Z12/30 Write block 186 len 36864 data: 13 13 16 14 12 14 12 10 10
144732024-02-27T21:49:36.494Z13/30 Read block 101 len 16384
144742024-02-27T21:49:36.494Z14/30 Write block 509 len 24576 data: 10 12 17 16 16 14
144752024-02-27T21:49:36.494Z15/30 Write block 353 len 4096 data: 10
144762024-02-27T21:49:36.494Z16/30 Read block 337 len 36864
144772024-02-27T21:49:36.497Z17/30 Read block 520 len 36864
144782024-02-27T21:49:36.497Z18/30 Read block 296 len 28672
144792024-02-27T21:49:36.500Z19/30 Write block 494 len 12288 data: 10 7 7
144802024-02-27T21:49:36.500Z20/30 Read block 586 len 24576
144812024-02-27T21:49:36.500Z21/30 Write block 106 len 28672 data: 14 16 15 13 12 13 11
144822024-02-27T21:49:36.500Z22/30 Read block 516 len 4096
144832024-02-27T21:49:36.504Z23/30 Read block 186 len 28672
144842024-02-27T21:49:36.504Z24/30 Write block 377 len 16384 data: 6 4 5 6
144852024-02-27T21:49:36.504Z25/30 Read block 199 len 4096
144862024-02-27T21:49:36.504Z26/30 Write block 141 len 36864 data: 10 11 10 9 8 9 7 8 8
144872024-02-27T21:49:36.508Z27/30 Write block 301 len 40960 data: 11 11 9 12 10 11 14 13 14 15
144882024-02-27T21:49:36.508Z28/30 Read block 228 len 16384
144892024-02-27T21:49:36.508Z29/30 Read block 524 len 8192
144902024-02-27T21:49:36.508Z30/30 Read block 155 len 40960
144912024-02-27T21:49:36.511Z----------------------------------------------------------------
144922024-02-27T21:49:36.511Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:27
144932024-02-27T21:49:36.511ZStates: Active Active Active
144942024-02-27T21:49:36.511ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144952024-02-27T21:49:36.512Z 4 Acked 1003 Write 10 Done Done Done false
144962024-02-27T21:49:36.512Z 5 Acked 1004 Read 8 Done Done Done false
144972024-02-27T21:49:36.512Z 6 Acked 1005 Read 3 Done Done Done false
144982024-02-27T21:49:36.512Z 7 Acked 1006 Read 8 Done Done Done false
144992024-02-27T21:49:36.512Z 8 Acked 1007 Read 8 Done Done Done false
145002024-02-27T21:49:36.512Z 9 Acked 1008 Write 5 Done Done Done false
145012024-02-27T21:49:36.512Z 10 Acked 1009 Write 2 Done Done Done false
145022024-02-27T21:49:36.512Z 11 Acked 1010 Read 3 Done Done Done false
145032024-02-27T21:49:36.512Z 12 Acked 1011 Write 9 Done Done Done false
145042024-02-27T21:49:36.512Z 13 Acked 1012 Read 4 Done Done Done false
145052024-02-27T21:49:36.512Z 14 Acked 1013 Write 6 Done Done Done false
145062024-02-27T21:49:36.512Z 15 Acked 1014 Write 1 Done Done Done false
145072024-02-27T21:49:36.512Z 16 Acked 1015 Read 9 Done Done Done false
145082024-02-27T21:49:36.512Z 17 Acked 1016 Read 9 Done Done Sent false
145092024-02-27T21:49:36.512Z 18 Acked 1017 Read 7 Done Done Sent false
145102024-02-27T21:49:36.512Z 19 Acked 1018 Write 3 Done Done Sent false
145112024-02-27T21:49:36.512Z 20 Acked 1019 Read 6 Done Done Sent false
145122024-02-27T21:49:36.512Z 21 Acked 1020 Write 7 Done Done Sent false
145132024-02-27T21:49:36.512Z 22 Acked 1021 Read 1 Done Done Sent false
145142024-02-27T21:49:36.512Z 23 Acked 1022 Read 7 Done Done Sent false
145152024-02-27T21:49:36.512Z 24 Acked 1023 Write 4 Done Done Sent false
145162024-02-27T21:49:36.512Z 25 Acked 1024 Read 1 Done Done Sent false
145172024-02-27T21:49:36.512Z 26 Acked 1025 Write 9 Done Done Sent false
145182024-02-27T21:49:36.512Z 27 Acked 1026 Write 10 Done Done Sent false
145192024-02-27T21:49:36.512Z 28 Acked 1027 Read 4 Done Done Sent false
145202024-02-27T21:49:36.513Z 29 Acked 1028 Read 2 Done Done Sent false
145212024-02-27T21:49:36.513Z 30 Acked 1029 Read 10 Done Sent Sent false
145222024-02-27T21:49:36.513Z STATES DS:0 DS:1 DS:2 TOTAL
145232024-02-27T21:49:36.513Z New 0 0 0 0
145242024-02-27T21:49:36.513Z Sent 0 1 14 15
145252024-02-27T21:49:36.513Z Done 27 26 13 66
145262024-02-27T21:49:36.513Z Skipped 0 0 0 0
145272024-02-27T21:49:36.513Z Error 0 0 0 0
145282024-02-27T21:49:36.513ZLast Flush: 1002 1002 1002
145292024-02-27T21:49:36.513ZDownstairs last five completed: 1002 1001 1000
145302024-02-27T21:49:36.513ZUpstairs last five completed: 30 29 28 27 26
145312024-02-27T21:49:36.513Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:34.880967349Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1683}
145322024-02-27T21:49:36.513ZWrote out file "/tmp/test_repair_verify.data"
145332024-02-27T21:49:36.516Z
145342024-02-27T21:49:36.516ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145352024-02-27T21:49:36.534ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145362024-02-27T21:49:36.534Z 0 000-019 132 132 132 248 248 248 F F F
145372024-02-27T21:49:36.535Z 1 020-039 132 132 132 248 248 248 F F F
145382024-02-27T21:49:36.535Z 2 040-059 124 124 124 232 232 232 F F F
145392024-02-27T21:49:36.535Z 3 060-079 130 130 130 244 244 244 F F F
145402024-02-27T21:49:36.535Z 4 080-099 124 124 124 229 229 229 F F F
145412024-02-27T21:49:36.535Z 5 100-119 132 132 132 247 247 247 T T F <---
145422024-02-27T21:49:36.535Z 6 120-139 130 130 130 242 242 242 F F F
145432024-02-27T21:49:36.535Z 7 140-159 128 128 128 240 240 240 T T F <---
145442024-02-27T21:49:36.535Z 8 160-179 128 128 128 240 240 240 F F F
145452024-02-27T21:49:36.535Z 9 180-199 130 130 130 243 243 243 T T T
145462024-02-27T21:49:36.538Z 10 200-219 129 129 129 241 241 241 F F F
145472024-02-27T21:49:36.538Z 11 220-239 132 132 132 247 247 247 F F F
145482024-02-27T21:49:36.538Z 12 240-259 133 133 133 249 249 249 F F F
145492024-02-27T21:49:36.538Z 13 260-279 133 133 133 249 249 249 F F F
145502024-02-27T21:49:36.538Z 14 280-299 132 132 132 246 246 246 F F F
145512024-02-27T21:49:36.538Z 15 300-319 132 132 132 246 246 246 T T F <---
145522024-02-27T21:49:36.539Z 16 320-339 129 129 129 241 241 241 T T T
145532024-02-27T21:49:36.539Z 17 340-359 133 133 133 249 249 249 T T T
145542024-02-27T21:49:36.539Z 18 360-379 130 130 130 243 243 243 T T F <---
145552024-02-27T21:49:36.539Z 19 380-399 130 130 130 243 243 243 T T F <---
145562024-02-27T21:49:36.539Z 20 400-419 134 134 134 250 250 250 F F F
145572024-02-27T21:49:36.539Z 21 420-439 134 134 134 250 250 250 F F F
145582024-02-27T21:49:36.539Z 22 440-459 134 134 134 250 250 250 F F F
145592024-02-27T21:49:36.539Z 23 460-479 132 132 132 247 247 247 F F F
145602024-02-27T21:49:36.539Z 24 480-499 130 130 130 242 242 242 T T F <---
145612024-02-27T21:49:36.539Z 25 500-519 132 132 132 246 246 246 T T T
145622024-02-27T21:49:36.539Z 26 520-539 127 127 127 237 237 237 T T T
145632024-02-27T21:49:36.539Z 27 540-559 129 129 129 241 241 241 F F F
145642024-02-27T21:49:36.539Z 28 560-579 129 129 129 241 241 241 F F F
145652024-02-27T21:49:36.539Z 29 580-599 131 131 131 245 245 245 F F F
145662024-02-27T21:49:36.539ZMax gen: 134, Max flush: 250
145672024-02-27T21:49:36.539ZError: Difference in extent metadata found!
145682024-02-27T21:49:36.539ZA Difference in extent metadata is expected here
145692024-02-27T21:49:36.539ZOn loop 67
145702024-02-27T21:49:36.539Z
145712024-02-27T21:49:36.543ZVerifying data now
145722024-02-27T21:49:42.124ZNo change after verify
145732024-02-27T21:49:42.124ZLoop: 67 Downstairs dump after verify (and repair):
145742024-02-27T21:49:42.138ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145752024-02-27T21:49:42.138Z 0 000-019 132 132 132 248 248 248 F F F
145762024-02-27T21:49:42.138Z 1 020-039 132 132 132 248 248 248 F F F
145772024-02-27T21:49:42.138Z 2 040-059 124 124 124 232 232 232 F F F
145782024-02-27T21:49:42.139Z 3 060-079 130 130 130 244 244 244 F F F
145792024-02-27T21:49:42.139Z 4 080-099 124 124 124 229 229 229 F F F
145802024-02-27T21:49:42.139Z 5 100-119 135 135 135 251 251 251 F F F
145812024-02-27T21:49:42.139Z 6 120-139 130 130 130 242 242 242 F F F
145822024-02-27T21:49:42.139Z 7 140-159 135 135 135 251 251 251 F F F
145832024-02-27T21:49:42.143Z 8 160-179 128 128 128 240 240 240 F F F
145842024-02-27T21:49:42.143Z 9 180-199 135 135 135 251 251 251 F F F
145852024-02-27T21:49:42.143Z 10 200-219 129 129 129 241 241 241 F F F
145862024-02-27T21:49:42.143Z 11 220-239 132 132 132 247 247 247 F F F
145872024-02-27T21:49:42.143Z 12 240-259 133 133 133 249 249 249 F F F
145882024-02-27T21:49:42.143Z 13 260-279 133 133 133 249 249 249 F F F
145892024-02-27T21:49:42.143Z 14 280-299 132 132 132 246 246 246 F F F
145902024-02-27T21:49:42.143Z 15 300-319 135 135 135 251 251 251 F F F
145912024-02-27T21:49:42.143Z 16 320-339 135 135 135 251 251 251 F F F
145922024-02-27T21:49:42.143Z 17 340-359 135 135 135 251 251 251 F F F
145932024-02-27T21:49:42.143Z 18 360-379 135 135 135 251 251 251 F F F
145942024-02-27T21:49:42.143Z 19 380-399 135 135 135 251 251 251 F F F
145952024-02-27T21:49:42.143Z 20 400-419 134 134 134 250 250 250 F F F
145962024-02-27T21:49:42.143Z 21 420-439 134 134 134 250 250 250 F F F
145972024-02-27T21:49:42.143Z 22 440-459 134 134 134 250 250 250 F F F
145982024-02-27T21:49:42.143Z 23 460-479 132 132 132 247 247 247 F F F
145992024-02-27T21:49:42.143Z 24 480-499 135 135 135 251 251 251 F F F
146002024-02-27T21:49:42.143Z 25 500-519 135 135 135 251 251 251 F F F
146012024-02-27T21:49:42.143Z 26 520-539 135 135 135 251 251 251 F F F
146022024-02-27T21:49:42.143Z 27 540-559 129 129 129 241 241 241 F F F
146032024-02-27T21:49:42.143Z 28 560-579 129 129 129 241 241 241 F F F
146042024-02-27T21:49:42.143Z 29 580-599 131 131 131 245 245 245 F F F
146052024-02-27T21:49:42.144ZMax gen: 135, Max flush: 251
146062024-02-27T21:49:42.144Z
146072024-02-27T21:49:42.144ZBegin loop 68
146082024-02-27T21:49:42.144ZDownstairs to restart: 1
146092024-02-27T21:49:42.161Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532205566Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691}
146102024-02-27T21:49:42.161Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532461925Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691}
146112024-02-27T21:49:42.161Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532479643Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691}
146122024-02-27T21:49:42.167Z{"msg":"Crucible fe7a130a-3525-42f8-9e86-37d4263dbf7c has session id: 427a7063-c43c-4882-92ca-023582886545","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.53250202Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146132024-02-27T21:49:42.167Z{"msg":"Upstairs opts: Upstairs UUID: fe7a130a-3525-42f8-9e86-37d4263dbf7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false"Crucible runtime is spawned
146142024-02-27T21:49:42.168Z,"v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532552474Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146152024-02-27T21:49:42.168Z{"msg":"Crucible stats registered with UUID: fe7a130a-3525-42f8-9e86-37d4263dbf7c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.53258699Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146162024-02-27T21:49:42.168Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532612827Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691}
146172024-02-27T21:49:42.168Z{"msg":"fe7a130a-3525-42f8-9e86-37d4263dbf7c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532791886Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146182024-02-27T21:49:42.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532824222Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146192024-02-27T21:49:42.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.53284449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146202024-02-27T21:49:42.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532867068Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146212024-02-27T21:49:42.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532881986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146222024-02-27T21:49:42.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532903253Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146232024-02-27T21:49:42.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.53292792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146242024-02-27T21:49:42.168Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532952027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146252024-02-27T21:49:42.168Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.532978055Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146262024-02-27T21:49:42.168Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.533001462Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146272024-02-27T21:49:42.168Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.533200658Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146282024-02-27T21:49:42.168Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.533250082Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146292024-02-27T21:49:42.168Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.53327512Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146302024-02-27T21:49:42.168Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.534000864Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146312024-02-27T21:49:42.168Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.534051728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146322024-02-27T21:49:42.168Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.534096173Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146332024-02-27T21:49:42.168Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.534138418Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146342024-02-27T21:49:42.168Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.534178453Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146352024-02-27T21:49:42.168Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.53420614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146362024-02-27T21:49:42.168Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.534639149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146372024-02-27T21:49:42.168Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.534686393Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146382024-02-27T21:49:42.168Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.53471598Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146392024-02-27T21:49:42.169Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535060409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146402024-02-27T21:49:42.169Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535110223Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146412024-02-27T21:49:42.169Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535159227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146422024-02-27T21:49:42.169Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535290062Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146432024-02-27T21:49:42.169Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535325468Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146442024-02-27T21:49:42.169Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535344706Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146452024-02-27T21:49:42.169Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535366903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146462024-02-27T21:49:42.169Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535383941Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146472024-02-27T21:49:42.169Z{"msg":"[0]R flush_numbers[0..12]: [248, 248, 232, 244, 229, 251, 242, 251, 240, 251, 241, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535408248Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146482024-02-27T21:49:42.169Z{"msg":"[0]R generation[0..12]: [132, 132, 124, 130, 124, 135, 130, 135, 128, 135, 129, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535443124Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146492024-02-27T21:49:42.169Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
146502024-02-27T21:49:42.169Z,"time":"2024-02-27T21:49:40.535470901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146512024-02-27T21:49:42.169Z{"msg":"[1]R flush_numbers[0..12]: [248, 248, 232, 244, 229, 251, 242, 251, 240, 251, 241, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535490319Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146522024-02-27T21:49:42.169Z{"msg":"[1]R generation[0..12]: [132, 132, 124, 130, 124, 135, 130, 135, 128, 135, 129, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535509506Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146532024-02-27T21:49:42.169Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535536983Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146542024-02-27T21:49:42.169Z{"msg":"[2]R flush_numbers[0..12]: [248, 248, 232, 244, 229, 251, 242, 251, 240, 251, 241, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535571509Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146552024-02-27T21:49:42.169Z{"msg":"[2]R generation[0..12]: [132, 132, 124, 130, 124, 135, 130, 135, 128, 135, 129, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535597496Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146562024-02-27T21:49:42.169Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535635811Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146572024-02-27T21:49:42.169Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535674177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146582024-02-27T21:49:42.169Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535710642Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146592024-02-27T21:49:42.169Z{"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535750328Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146602024-02-27T21:49:42.169Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535787684Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"":"downstairs","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146612024-02-27T21:49:42.169Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146622024-02-27T21:49:42.169Z,"time":"2024-02-27T21:49:40.535827879Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146632024-02-27T21:49:42.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535863025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"0","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146642024-02-27T21:49:42.170Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535887682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"1","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146652024-02-27T21:49:42.170Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.53590177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"client":"2","":"downstairs","session_id":"427a7063-c43c-4882-92ca-023582886545"}
146662024-02-27T21:49:42.170Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535922578Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146672024-02-27T21:49:42.170Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535935416Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146682024-02-27T21:49:42.170Z{"msg":"fe7a130a-3525-42f8-9e86-37d4263dbf7c is now active with session: 427a7063-c43c-4882-92ca-023582886545","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535955844Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146692024-02-27T21:49:42.170Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.535968182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691,"session_id":"427a7063-c43c-4882-92ca-023582886545"}
146702024-02-27T21:49:42.170Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:40.53598704Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691}
146712024-02-27T21:49:42.173ZLoading write count information from file "/tmp/test_repair_verify.data"
146722024-02-27T21:49:42.173ZRun Repair workload
146732024-02-27T21:49:42.173Z01/30 Write block 185 len 8192 data: 13 14
146742024-02-27T21:49:42.173Z02/30 Read block 258 len 28672
146752024-02-27T21:49:42.174Z03/30 Write block 321 len 28672 data: 14 12 11 10 8 6 8
146762024-02-27T21:49:42.174Z04/30 Read block 99 len 40960
146772024-02-27T21:49:42.174Z05/30 Write block 21 len 12288 data: 10 7 5
146782024-02-27T21:49:42.174Z06/30 Read block 92 len 8192
146792024-02-27T21:49:42.174Z07/30 Write block 254 len 8192 data: 10 14
146802024-02-27T21:49:42.174Z08/30 Write block 469 len 4096 data: 11
146812024-02-27T21:49:42.174Z09/30 Write block 316 len 8192 data: 16 15
146822024-02-27T21:49:42.174Z10/30 Read block 459 len 24576
146832024-02-27T21:49:42.174Z11/30 Read block 28 len 12288
146842024-02-27T21:49:42.174Z12/30 Read block 259 len 20480
146852024-02-27T21:49:42.177Z13/30 Write block 412 len 36864 data: 10 11 11 12 13 17 15 14 9
146862024-02-27T21:49:42.177Z14/30 Read block 490 len 40960
146872024-02-27T21:49:42.177Z15/30 Read block 461 len 4096
146882024-02-27T21:49:42.178Z16/30 Read block 545 len 36864
146892024-02-27T21:49:42.181Z17/30 Read block 88 len 4096
146902024-02-27T21:49:42.181Z18/30 Read block 305 len 36864
146912024-02-27T21:49:42.181Z19/30 Read block 413 len 36864
146922024-02-27T21:49:42.181Z20/30 Read block 16 len 32768
146932024-02-27T21:49:42.184Z21/30 Read block 20 len 20480
146942024-02-27T21:49:42.184Z22/30 Write block 362 len 8192 data: 6 10
146952024-02-27T21:49:42.185Z23/30 Write block 156 len 20480 data: 12 12 12 12 12
146962024-02-27T21:49:42.185Z24/30 Flush
146972024-02-27T21:49:42.187Z25/30 Write block 290 len 36864 data: 11 11 10 13 13 14 14 14 13
146982024-02-27T21:49:42.191Z26/30 Write block 93 len 4096 data: 10
146992024-02-27T21:49:42.191Z27/30 Read block 35 len 16384
147002024-02-27T21:49:42.191Z28/30 Write block 207 len 8192 data: 6 8
147012024-02-27T21:49:42.191Z29/30 Write block 562 len 36864 data: 15 14 11 9 7 6 9 9 7
147022024-02-27T21:49:42.191Z30/30 Read block 69 len 32768
147032024-02-27T21:49:42.194Z----------------------------------------------------------------
147042024-02-27T21:49:42.194Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
147052024-02-27T21:49:42.194ZStates: Active Active Active
147062024-02-27T21:49:42.194ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
147072024-02-27T21:49:42.195Z 1 Acked 1000 Write 2 Done Done Done false
147082024-02-27T21:49:42.195Z 2 Acked 1001 Read 7 Done Sent Done false
147092024-02-27T21:49:42.195Z 3 Acked 1002 Write 7 Done Sent Done false
147102024-02-27T21:49:42.195Z 4 Acked 1003 Read 10 Done Sent Done false
147112024-02-27T21:49:42.195Z 5 Acked 1004 Write 3 Done Sent Done false
147122024-02-27T21:49:42.195Z 6 Acked 1005 Read 2 Done Sent Done false
147132024-02-27T21:49:42.195Z 7 Acked 1006 Write 2 Done Sent Done false
147142024-02-27T21:49:42.195Z 8 Acked 1007 Write 1 Done Sent Done false
147152024-02-27T21:49:42.195Z 9 Acked 1008 Write 2 Done Sent Done false
147162024-02-27T21:49:42.195Z 10 Acked 1009 Read 6 Done Sent Done false
147172024-02-27T21:49:42.195Z 11 Acked 1010 Read 3 Done Sent Done false
147182024-02-27T21:49:42.195Z 12 Acked 1011 Read 5 Done Sent Done false
147192024-02-27T21:49:42.195Z 13 Acked 1012 Write 9 Done Sent Done false
147202024-02-27T21:49:42.195Z 14 Acked 1013 Read 10 Done Sent Done false
147212024-02-27T21:49:42.195Z 15 Acked 1014 Read 1 Done Sent Done false
147222024-02-27T21:49:42.195Z 16 Acked 1015 Read 9 Done Sent Done false
147232024-02-27T21:49:42.195Z 17 Acked 1016 Read 1 Done Sent Done false
147242024-02-27T21:49:42.195Z 18 Acked 1017 Read 9 Done Sent Done false
147252024-02-27T21:49:42.195Z 19 Acked 1018 Read 9 Done Sent Done false
147262024-02-27T21:49:42.195Z 20 Acked 1019 Read 8 Done Sent Done false
147272024-02-27T21:49:42.195Z 21 Acked 1020 Read 5 Done Sent Done false
147282024-02-27T21:49:42.195Z 22 Acked 1021 Write 2 Done Sent Done false
147292024-02-27T21:49:42.195Z 23 Acked 1022 Write 5 Done Sent Done false
147302024-02-27T21:49:42.195Z 24 Acked 1023 Flush 0 Done Sent Done false
147312024-02-27T21:49:42.195Z 25 Acked 1024 Write 9 Done Sent Done false
147322024-02-27T21:49:42.196Z 26 Acked 1025 Write 1 Done Sent Done false
147332024-02-27T21:49:42.196Z 27 Acked 1026 Read 4 Done Sent Done false
147342024-02-27T21:49:42.196Z 28 Acked 1027 Write 2 Done Sent Done false
147352024-02-27T21:49:42.196Z 29 Acked 1028 Write 9 Done Sent Done false
147362024-02-27T21:49:42.196Z 30 Acked 1029 Read 8 Sent Sent Done false
147372024-02-27T21:49:42.196Z STATES DS:0 DS:1 DS:2 TOTAL
147382024-02-27T21:49:42.196Z New 0 0 0 0
147392024-02-27T21:49:42.196Z Sent 1 29 0 30
147402024-02-27T21:49:42.196Z Done 29 1 30 60
147412024-02-27T21:49:42.196Z Skipped 0 0 0 0
147422024-02-27T21:49:42.196Z Error 0 0 0 0
147432024-02-27T21:49:42.196ZLast Flush: 1023 0 1023
147442024-02-27T21:49:42.196ZDownstairs last five completed:
147452024-02-27T21:49:42.196ZUpstairs last five completed: 30 29 28 27 26
147462024-02-27T21:49:42.196Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:40.564413651Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1691}
147472024-02-27T21:49:42.196ZWrote out file "/tmp/test_repair_verify.data"
147482024-02-27T21:49:42.199Z
147492024-02-27T21:49:42.202ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147502024-02-27T21:49:42.219ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147512024-02-27T21:49:42.219Z 0 000-019 132 132 132 248 248 248 F F F
147522024-02-27T21:49:42.219Z 1 020-039 136 132 136 252 248 252 F F F <---
147532024-02-27T21:49:42.219Z 2 040-059 124 124 124 232 232 232 F F F
147542024-02-27T21:49:42.219Z 3 060-079 130 130 130 244 244 244 F F F
147552024-02-27T21:49:42.219Z 4 080-099 124 124 124 229 229 229 T F T <---
147562024-02-27T21:49:42.223Z 5 100-119 135 135 135 251 251 251 F F F
147572024-02-27T21:49:42.223Z 6 120-139 130 130 130 242 242 242 F F F
147582024-02-27T21:49:42.223Z 7 140-159 136 135 136 252 251 252 F F F <---
147592024-02-27T21:49:42.223Z 8 160-179 136 128 136 252 240 252 F F F <---
147602024-02-27T21:49:42.223Z 9 180-199 136 135 136 252 251 252 F T F <---
147612024-02-27T21:49:42.223Z 10 200-219 129 129 129 241 241 241 T F T <---
147622024-02-27T21:49:42.223Z 11 220-239 132 132 132 247 247 247 F F F
147632024-02-27T21:49:42.223Z 12 240-259 136 133 136 252 249 252 F F F <---
147642024-02-27T21:49:42.223Z 13 260-279 133 133 133 249 249 249 F F F
147652024-02-27T21:49:42.223Z 14 280-299 132 132 132 246 246 246 T F T <---
147662024-02-27T21:49:42.223Z 15 300-319 136 135 136 252 251 252 F F F <---
147672024-02-27T21:49:42.223Z 16 320-339 136 135 136 252 251 252 F F F <---
147682024-02-27T21:49:42.223Z 17 340-359 135 135 135 251 251 251 F F F
147692024-02-27T21:49:42.223Z 18 360-379 136 135 136 252 251 252 F F F <---
147702024-02-27T21:49:42.223Z 19 380-399 135 135 135 251 251 251 F F F
147712024-02-27T21:49:42.223Z 20 400-419 136 134 136 252 250 252 F F F <---
147722024-02-27T21:49:42.223Z 21 420-439 136 134 136 252 250 252 F F F <---
147732024-02-27T21:49:42.223Z 22 440-459 134 134 134 250 250 250 F F F
147742024-02-27T21:49:42.223Z 23 460-479 136 132 136 252 247 252 F F F <---
147752024-02-27T21:49:42.223Z 24 480-499 135 135 135 251 251 251 F F F
147762024-02-27T21:49:42.223Z 25 500-519 135 135 135 251 251 251 F F F
147772024-02-27T21:49:42.223Z 26 520-539 135 135 135 251 251 251 F F F
147782024-02-27T21:49:42.224Z 27 540-559 129 129 129 241 241 241 F F F
147792024-02-27T21:49:42.224Z 28 560-579 129 129 129 241 241 241 T F T <---
147802024-02-27T21:49:42.224Z 29 580-599 131 131 131 245 245 245 F F F
147812024-02-27T21:49:42.224ZMax gen: 136, Max flush: 252
147822024-02-27T21:49:42.224ZError: Difference in extent metadata found!
147832024-02-27T21:49:42.224ZA Difference in extent metadata is expected here
147842024-02-27T21:49:42.224ZOn loop 68
147852024-02-27T21:49:42.224Z
147862024-02-27T21:49:42.224ZVerifying data now
147872024-02-27T21:49:46.960ZNo change after verify
147882024-02-27T21:49:46.961ZLoop: 68 Downstairs dump after verify (and repair):
147892024-02-27T21:49:46.974ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147902024-02-27T21:49:46.974Z 0 000-019 132 132 132 248 248 248 F F F
147912024-02-27T21:49:46.974Z 1 020-039 136 136 136 252 252 252 F F F
147922024-02-27T21:49:46.974Z 2 040-059 124 124 124 232 232 232 F F F
147932024-02-27T21:49:46.974Z 3 060-079 130 130 130 244 244 244 F F F
147942024-02-27T21:49:46.974Z 4 080-099 137 137 137 253 253 253 F F F
147952024-02-27T21:49:46.974Z 5 100-119 135 135 135 251 251 251 F F F
147962024-02-27T21:49:46.978Z 6 120-139 130 130 130 242 242 242 F F F
147972024-02-27T21:49:46.978Z 7 140-159 136 136 136 252 252 252 F F F
147982024-02-27T21:49:46.978Z 8 160-179 136 136 136 252 252 252 F F F
147992024-02-27T21:49:46.978Z 9 180-199 136 136 136 252 252 252 F F F
148002024-02-27T21:49:46.978Z 10 200-219 137 137 137 253 253 253 F F F
148012024-02-27T21:49:46.978Z 11 220-239 132 132 132 247 247 247 F F F
148022024-02-27T21:49:46.978Z 12 240-259 136 136 136 252 252 252 F F F
148032024-02-27T21:49:46.978Z 13 260-279 133 133 133 249 249 249 F F F
148042024-02-27T21:49:46.978Z 14 280-299 137 137 137 253 253 253 F F F
148052024-02-27T21:49:46.978Z 15 300-319 136 136 136 252 252 252 F F F
148062024-02-27T21:49:46.978Z 16 320-339 136 136 136 252 252 252 F F F
148072024-02-27T21:49:46.978Z 17 340-359 135 135 135 251 251 251 F F F
148082024-02-27T21:49:46.978Z 18 360-379 136 136 136 252 252 252 F F F
148092024-02-27T21:49:46.978Z 19 380-399 135 135 135 251 251 251 F F F
148102024-02-27T21:49:46.978Z 20 400-419 136 136 136 252 252 252 F F F
148112024-02-27T21:49:46.978Z 21 420-439 136 136 136 252 252 252 F F F
148122024-02-27T21:49:46.978Z 22 440-459 134 134 134 250 250 250 F F F
148132024-02-27T21:49:46.978Z 23 460-479 136 136 136 252 252 252 F F F
148142024-02-27T21:49:46.978Z 24 480-499 135 135 135 251 251 251 F F F
148152024-02-27T21:49:46.979Z 25 500-519 135 135 135 251 251 251 F F F
148162024-02-27T21:49:46.979Z 26 520-539 135 135 135 251 251 251 F F F
148172024-02-27T21:49:46.979Z 27 540-559 129 129 129 241 241 241 F F F
148182024-02-27T21:49:46.979Z 28 560-579 137 137 137 253 253 253 F F F
148192024-02-27T21:49:46.979Z 29 580-599 131 131 131 245 245 245 F F F
148202024-02-27T21:49:46.979ZMax gen: 137, Max flush: 253
148212024-02-27T21:49:46.979Z
148222024-02-27T21:49:46.979ZBegin loop 69
148232024-02-27T21:49:46.979ZDownstairs to restart: 2
148242024-02-27T21:49:46.996Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.36659997Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699}
148252024-02-27T21:49:46.996Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.366866419Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699}
148262024-02-27T21:49:46.996Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.366884797Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699}
148272024-02-27T21:49:47.003Z{"msg":"Crucible 401a5edb-ad98-456e-b931-df9a9c26138a has session id: a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.366907454Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148282024-02-27T21:49:47.003Z{"msg":"Upstairs opts: Upstairs UUID: 401a5edb-ad98-456e-b931-df9a9c26138a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.36693956Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148292024-02-27T21:49:47.003ZCrucible runtime is spawned
148302024-02-27T21:49:47.003Z{"msg":"Crucible stats registered with UUID: 401a5edb-ad98-456e-b931-df9a9c26138a","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.366988794Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148312024-02-27T21:49:47.003Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367096792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699}
148322024-02-27T21:49:47.003Z{"msg":"401a5edb-ad98-456e-b931-df9a9c26138a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367165903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148332024-02-27T21:49:47.003Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367218297Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148342024-02-27T21:49:47.003Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367306477Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148352024-02-27T21:49:47.003Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367328934Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148362024-02-27T21:49:47.004Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367343763Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148372024-02-27T21:49:47.004Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.36736098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148382024-02-27T21:49:47.004Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367407815Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148392024-02-27T21:49:47.004Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367435432Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148402024-02-27T21:49:47.004Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367469868Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148412024-02-27T21:49:47.004Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367491555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148422024-02-27T21:49:47.004Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367515312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148432024-02-27T21:49:47.004Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367540319Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148442024-02-27T21:49:47.004Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.367622439Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148452024-02-27T21:49:47.004Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.368200202Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148462024-02-27T21:49:47.004Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.368236237Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148472024-02-27T21:49:47.004Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.368254135Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148482024-02-27T21:49:47.004Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.368287451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148492024-02-27T21:49:47.004Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.368350294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148502024-02-27T21:49:47.005Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.368391989Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148512024-02-27T21:49:47.005Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.368864303Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148522024-02-27T21:49:47.005Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.368916737Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148532024-02-27T21:49:47.005Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.368964632Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148542024-02-27T21:49:47.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369501618Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148552024-02-27T21:49:47.005Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369537414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148562024-02-27T21:49:47.005Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369552152Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148572024-02-27T21:49:47.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369582049Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148582024-02-27T21:49:47.005Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369616825Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148592024-02-27T21:49:47.005Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369649511Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148602024-02-27T21:49:47.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369680737Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148612024-02-27T21:49:47.005ZWait for a query_work_queue command to finish before sending IO
148622024-02-27T21:49:47.005Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369703345Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148632024-02-27T21:49:47.006Z{"msg":"[0]R flush_numbers[0..12]: [248, 252, 232, 244, 253, 251, 242, 252, 252, 252, 253, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369726442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148642024-02-27T21:49:47.006Z{"msg":"[0]R generation[0..12]: [132, 136, 124, 130, 137, 135, 130, 136, 136, 136, 137, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.3697459Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148652024-02-27T21:49:47.006Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369768457Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148662024-02-27T21:49:47.006Z{"msg":"[1]R flush_numbers[0..12]: [248, 252, 232, 244, 253, 251, 242, 252, 252, 252, 253, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369786805Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148672024-02-27T21:49:47.006Z{"msg":"[1]R generation[0..12]: [132, 136, 124, 130, 137, 135, 130, 136, 136, 136, 137, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369817481Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148682024-02-27T21:49:47.006Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369833519Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148692024-02-27T21:49:47.006Z{"msg":"[2]R flush_numbers[0..12]: [248, 252, 232, 244, 253, 251, 242, 252, 252, 252, 253, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369865195Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148702024-02-27T21:49:47.006Z{"msg":"[2]R generation[0..12]: [132, 136, 124, 130, 137, 135, 130, 136, 136, 136, 137, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369885203Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148712024-02-27T21:49:47.006Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.36991464Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148722024-02-27T21:49:47.006Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369933527Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148732024-02-27T21:49:47.006Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369962234Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148742024-02-27T21:49:47.006Z{"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.369977152Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148752024-02-27T21:49:47.006Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370007089Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"":"downstairs","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148762024-02-27T21:49:47.007Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370022177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148772024-02-27T21:49:47.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370050054Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"0","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148782024-02-27T21:49:47.007ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148792024-02-27T21:49:47.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370087689Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"1","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148802024-02-27T21:49:47.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370120025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"client":"2","":"downstairs","session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148812024-02-27T21:49:47.007Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370149252Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148822024-02-27T21:49:47.007Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370181458Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148832024-02-27T21:49:47.007Z{"msg":"401a5edb-ad98-456e-b931-df9a9c26138a is now active with session: a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370213354Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148842024-02-27T21:49:47.007Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370243751Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699,"session_id":"a4ce6b65-2006-42c3-9d22-ca5a4a6e5f25"}
148852024-02-27T21:49:47.007Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:45.370274097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699}
148862024-02-27T21:49:47.007ZLoading write count information from file "/tmp/test_repair_verify.data"
148872024-02-27T21:49:47.007ZRun Repair workload
148882024-02-27T21:49:47.011Z01/30 Write block 372 len 24576 data: 4 5 4 5 5 7
148892024-02-27T21:49:47.011Z02/30 Write block 555 len 4096 data: 12
148902024-02-27T21:49:47.012Z03/30 Read block 546 len 24576
148912024-02-27T21:49:47.012Z04/30 Write block 592 len 20480 data: 11 11 9 8 9
148922024-02-27T21:49:47.012Z05/30 Write block 123 len 4096 data: 14
148932024-02-27T21:49:47.012Z06/30 Write block 372 len 36864 data: 5 6 5 6 6 8 5 6 7
148942024-02-27T21:49:47.012Z07/30 Write block 501 len 28672 data: 6 7 7 8 9 10 8
148952024-02-27T21:49:47.012Z08/30 Read block 59 len 8192
148962024-02-27T21:49:47.012Z09/30 Read block 533 len 36864
148972024-02-27T21:49:47.012Z10/30 Flush
148982024-02-27T21:49:47.012Z11/30 Write block 341 len 36864 data: 16 14 13 13 11 11 11 15 16
148992024-02-27T21:49:47.031Z12/30 Write block 21 len 32768 data: 11 8 6 10 9 9 9 9
149002024-02-27T21:49:47.031Z13/30 Read block 438 len 32768
149012024-02-27T21:49:47.031Z14/30 Read block 24 len 40960
149022024-02-27T21:49:47.032Z15/30 Read block 330 len 20480
149032024-02-27T21:49:47.032Z16/30 Read block 471 len 4096
149042024-02-27T21:49:47.044Z17/30 Write block 8 len 28672 data: 12 14 15 15 16 14 13
149052024-02-27T21:49:47.044Z18/30 Read block 290 len 8192
149062024-02-27T21:49:47.044Z19/30 Flush
149072024-02-27T21:49:47.044Z20/30 Write block 118 len 24576 data: 10 11 13 15 13 15
149082024-02-27T21:49:47.044Z21/30 Read block 498 len 8192
149092024-02-27T21:49:47.044Z22/30 Read block 554 len 32768
149102024-02-27T21:49:47.044Z23/30 Read block 118 len 24576
149112024-02-27T21:49:47.044Z24/30 Write block 340 len 4096 data: 16
149122024-02-27T21:49:47.044Z25/30 Read block 124 len 36864
149132024-02-27T21:49:47.044Z26/30 Write block 261 len 24576 data: 11 11 10 11 11 7
149142024-02-27T21:49:47.045Z27/30 Write block 159 len 20480 data: 13 13 12 11 11
149152024-02-27T21:49:47.045Z28/30 Write block 329 len 28672 data: 10 10 11 11 13 13 16
149162024-02-27T21:49:47.045Z29/30 Read block 268 len 8192
149172024-02-27T21:49:47.045Z30/30 Write block 448 len 16384 data: 7 8 9 10
149182024-02-27T21:49:47.045Z----------------------------------------------------------------
149192024-02-27T21:49:47.045Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
149202024-02-27T21:49:47.045ZStates: Active Active Active
149212024-02-27T21:49:47.045ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149222024-02-27T21:49:47.045Z 1 Acked 1000 Write 6 Done Done Done false
149232024-02-27T21:49:47.045Z 2 Acked 1001 Write 1 Done Done Done false
149242024-02-27T21:49:47.045Z 3 Acked 1002 Read 6 Done Done Done false
149252024-02-27T21:49:47.045Z 4 Acked 1003 Write 5 Done Done Done false
149262024-02-27T21:49:47.045Z 5 Acked 1004 Write 1 Done Done Done false
149272024-02-27T21:49:47.045Z 6 Acked 1005 Write 9 Done Done Done false
149282024-02-27T21:49:47.045Z 7 Acked 1006 Write 7 Done Done Sent false
149292024-02-27T21:49:47.045Z 8 Acked 1007 Read 2 Done Done Sent false
149302024-02-27T21:49:47.045Z 9 Acked 1008 Read 9 Done Done Sent false
149312024-02-27T21:49:47.045Z 10 Acked 1009 Flush 0 Done Done Sent false
149322024-02-27T21:49:47.045Z 11 Acked 1010 Write 9 Done Done Sent false
149332024-02-27T21:49:47.045Z 12 Acked 1011 Write 8 Done Done Sent false
149342024-02-27T21:49:47.045Z 13 Acked 1012 Read 8 Done Done Sent false
149352024-02-27T21:49:47.045Z 14 Acked 1013 Read 10 Done Done Sent false
149362024-02-27T21:49:47.045Z 15 Acked 1014 Read 5 Done Done Sent false
149372024-02-27T21:49:47.046Z 16 Acked 1015 Read 1 Done Done Sent false
149382024-02-27T21:49:47.046Z 17 Acked 1016 Write 7 Done Done Sent false
149392024-02-27T21:49:47.046Z 18 Acked 1017 Read 2 Done Done Sent false
149402024-02-27T21:49:47.046Z 19 Acked 1018 Flush 0 Done Done Sent false
149412024-02-27T21:49:47.046Z 20 Acked 1019 Write 6 Done Done Sent false
149422024-02-27T21:49:47.046Z 21 Acked 1020 Read 2 Done Done Sent false
149432024-02-27T21:49:47.046Z 22 Acked 1021 Read 8 Done Done Sent false
149442024-02-27T21:49:47.046Z 23 Acked 1022 Read 6 Done Done Sent false
149452024-02-27T21:49:47.046Z 24 Acked 1023 Write 1 Done Done Sent false
149462024-02-27T21:49:47.046Z 25 Acked 1024 Read 9 Done Done Sent false
149472024-02-27T21:49:47.046Z 26 Acked 1025 Write 6 Done Done Sent false
149482024-02-27T21:49:47.046Z 27 Acked 1026 Write 5 Done Done Sent false
149492024-02-27T21:49:47.047Z 28 Acked 1027 Write 7 Done Done Sent false
149502024-02-27T21:49:47.047Z 29 Acked 1028 Read 2 Done Done Sent false
149512024-02-27T21:49:47.047Z 30 Acked 1029 Write 4 Sent Sent Sent false
149522024-02-27T21:49:47.047Z STATES DS:0 DS:1 DS:2 TOTAL
149532024-02-27T21:49:47.047Z New 0 0 0 0
149542024-02-27T21:49:47.047Z Sent 1 1 24 26
149552024-02-27T21:49:47.047Z Done 29 29 6 64
149562024-02-27T21:49:47.047Z Skipped 0 0 0 0
149572024-02-27T21:49:47.047Z Error 0 0 0 0
149582024-02-27T21:49:47.048ZLast Flush: 1018 1018 0
149592024-02-27T21:49:47.048ZDownstairs last five completed:
149602024-02-27T21:49:47.048ZUpstairs last five completed: 30 29 28 27 26
149612024-02-27T21:49:47.048Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:45.400378151Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1699}
149622024-02-27T21:49:47.048ZWrote out file "/tmp/test_repair_verify.data"
149632024-02-27T21:49:47.048Z
149642024-02-27T21:49:47.054ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149652024-02-27T21:49:47.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149662024-02-27T21:49:47.058Z 0 000-019 138 138 132 255 255 248 F F F <---
149672024-02-27T21:49:47.058Z 1 020-039 138 138 136 255 255 252 F F F <---
149682024-02-27T21:49:47.058Z 2 040-059 124 124 124 232 232 232 F F F
149692024-02-27T21:49:47.058Z 3 060-079 130 130 130 244 244 244 F F F
149702024-02-27T21:49:47.058Z 4 080-099 137 137 137 253 253 253 F F F
149712024-02-27T21:49:47.058Z 5 100-119 135 135 135 251 251 251 T T F <---
149722024-02-27T21:49:47.058Z 6 120-139 138 138 130 254 254 242 T T T <---
149732024-02-27T21:49:47.058Z 7 140-159 136 136 136 252 252 252 T T F <---
149742024-02-27T21:49:47.058Z 8 160-179 136 136 136 252 252 252 T T F <---
149752024-02-27T21:49:47.058Z 9 180-199 136 136 136 252 252 252 F F F
149762024-02-27T21:49:47.058Z 10 200-219 137 137 137 253 253 253 F F F
149772024-02-27T21:49:47.058Z 11 220-239 132 132 132 247 247 247 F F F
149782024-02-27T21:49:47.058Z 12 240-259 136 136 136 252 252 252 F F F
149792024-02-27T21:49:47.058Z 13 260-279 133 133 133 249 249 249 T T F <---
149802024-02-27T21:49:47.058Z 14 280-299 137 137 137 253 253 253 F F F
149812024-02-27T21:49:47.058Z 15 300-319 136 136 136 252 252 252 F F F
149822024-02-27T21:49:47.058Z 16 320-339 136 136 136 252 252 252 T T F <---
149832024-02-27T21:49:47.058Z 17 340-359 138 138 135 255 255 251 T T F <---
149842024-02-27T21:49:47.059Z 18 360-379 138 138 136 254 254 252 F F T <---
149852024-02-27T21:49:47.059Z 19 380-399 138 138 135 254 254 251 F F T <---
149862024-02-27T21:49:47.059Z 20 400-419 136 136 136 252 252 252 F F F
149872024-02-27T21:49:47.059Z 21 420-439 136 136 136 252 252 252 F F F
149882024-02-27T21:49:47.059Z 22 440-459 134 134 134 250 250 250 T T F <---
149892024-02-27T21:49:47.059Z 23 460-479 136 136 136 252 252 252 F F F
149902024-02-27T21:49:47.059Z 24 480-499 135 135 135 251 251 251 F F F
149912024-02-27T21:49:47.059Z 25 500-519 138 138 135 254 254 251 F F F <---
149922024-02-27T21:49:47.059Z 26 520-539 135 135 135 251 251 251 F F F
149932024-02-27T21:49:47.059Z 27 540-559 138 138 129 254 254 241 F F T <---
149942024-02-27T21:49:47.059Z 28 560-579 137 137 137 253 253 253 F F F
149952024-02-27T21:49:47.059Z 29 580-599 138 138 131 254 254 245 F F T <---
149962024-02-27T21:49:47.059ZMax gen: 138, Max flush: 255
149972024-02-27T21:49:47.059ZError: Difference in extent metadata found!
149982024-02-27T21:49:47.059ZA Difference in extent metadata is expected here
149992024-02-27T21:49:47.059ZOn loop 69
150002024-02-27T21:49:47.059Z
150012024-02-27T21:49:47.063ZVerifying data now
150022024-02-27T21:49:52.804ZNo change after verify
150032024-02-27T21:49:52.804ZLoop: 69 Downstairs dump after verify (and repair):
150042024-02-27T21:49:52.818ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150052024-02-27T21:49:52.818Z 0 000-019 138 138 138 255 255 255 F F F
150062024-02-27T21:49:52.819Z 1 020-039 138 138 138 255 255 255 F F F
150072024-02-27T21:49:52.819Z 2 040-059 124 124 124 232 232 232 F F F
150082024-02-27T21:49:52.819Z 3 060-079 130 130 130 244 244 244 F F F
150092024-02-27T21:49:52.819Z 4 080-099 137 137 137 253 253 253 F F F
150102024-02-27T21:49:52.819Z 5 100-119 139 139 139 256 256 256 F F F
150112024-02-27T21:49:52.819Z 6 120-139 139 139 139 256 256 256 F F F
150122024-02-27T21:49:52.819Z 7 140-159 139 139 139 256 256 256 F F F
150132024-02-27T21:49:52.819Z 8 160-179 139 139 139 256 256 256 F F F
150142024-02-27T21:49:52.819Z 9 180-199 136 136 136 252 252 252 F F F
150152024-02-27T21:49:52.819Z 10 200-219 137 137 137 253 253 253 F F F
150162024-02-27T21:49:52.819Z 11 220-239 132 132 132 247 247 247 F F F
150172024-02-27T21:49:52.819Z 12 240-259 136 136 136 252 252 252 F F F
150182024-02-27T21:49:52.819Z 13 260-279 139 139 139 256 256 256 F F F
150192024-02-27T21:49:52.819Z 14 280-299 137 137 137 253 253 253 F F F
150202024-02-27T21:49:52.819Z 15 300-319 136 136 136 252 252 252 F F F
150212024-02-27T21:49:52.819Z 16 320-339 139 139 139 256 256 256 F F F
150222024-02-27T21:49:52.829Z 17 340-359 139 139 139 256 256 256 F F F
150232024-02-27T21:49:52.829Z 18 360-379 138 138 138 254 254 254 F F F
150242024-02-27T21:49:52.830Z 19 380-399 138 138 138 254 254 254 F F F
150252024-02-27T21:49:52.830Z 20 400-419 136 136 136 252 252 252 F F F
150262024-02-27T21:49:52.830Z 21 420-439 136 136 136 252 252 252 F F F
150272024-02-27T21:49:52.830Z 22 440-459 139 139 139 256 256 256 F F F
150282024-02-27T21:49:52.830Z 23 460-479 136 136 136 252 252 252 F F F
150292024-02-27T21:49:52.830Z 24 480-499 135 135 135 251 251 251 F F F
150302024-02-27T21:49:52.830Z 25 500-519 138 138 138 254 254 254 F F F
150312024-02-27T21:49:52.830Z 26 520-539 135 135 135 251 251 251 F F F
150322024-02-27T21:49:52.831Z 27 540-559 138 138 138 254 254 254 F F F
150332024-02-27T21:49:52.831Z 28 560-579 137 137 137 253 253 253 F F F
150342024-02-27T21:49:52.831Z 29 580-599 138 138 138 254 254 254 F F F
150352024-02-27T21:49:52.831ZMax gen: 139, Max flush: 256
150362024-02-27T21:49:52.831Z
150372024-02-27T21:49:52.831ZBegin loop 70
150382024-02-27T21:49:52.831ZDownstairs to restart: 2
150392024-02-27T21:49:52.845Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.212521625Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707}
150402024-02-27T21:49:52.846Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.212880613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707}
150412024-02-27T21:49:52.846Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.212909539Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707}
150422024-02-27T21:49:52.846Z{"msg":"Crucible 85be349b-bcaf-4fa1-909e-f6ac46ddd87b has session id: 98be17fd-938f-488f-8e6b-455657d2e753"Crucible runtime is spawned
150432024-02-27T21:49:52.966Z,"v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.212952724Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150442024-02-27T21:49:52.966Z{"msg":"Upstairs opts: Upstairs UUID: 85be349b-bcaf-4fa1-909e-f6ac46ddd87b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213000838Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150452024-02-27T21:49:52.966Z{"msg":"Crucible stats registered with UUID: 85be349b-bcaf-4fa1-909e-f6ac46ddd87b","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213036974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150462024-02-27T21:49:52.966Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.21307077Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707}
150472024-02-27T21:49:52.966Z{"msg":"85be349b-bcaf-4fa1-909e-f6ac46ddd87b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213108016Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150482024-02-27T21:49:52.966Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213148901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150492024-02-27T21:49:52.966Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213214193Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150502024-02-27T21:49:52.966Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.21324153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150512024-02-27T21:49:52.966Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213267307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150522024-02-27T21:49:52.966Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213292244Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150532024-02-27T21:49:52.966Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213331859Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150542024-02-27T21:49:52.966Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213360696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150552024-02-27T21:49:52.966Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.21340939Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150562024-02-27T21:49:52.966Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.21350004Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150572024-02-27T21:49:52.966Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213573601Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150582024-02-27T21:49:52.967Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213610947Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150592024-02-27T21:49:52.967Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.213639903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150602024-02-27T21:49:52.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.214130156Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150612024-02-27T21:49:52.967Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.214187379Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150622024-02-27T21:49:52.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.214217005Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150632024-02-27T21:49:52.967Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.214253561Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150642024-02-27T21:49:52.967Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.214730995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150652024-02-27T21:49:52.967Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.21494075Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150662024-02-27T21:49:52.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.215142296Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150672024-02-27T21:49:52.967Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.215182152Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150682024-02-27T21:49:52.967Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.215305877Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150692024-02-27T21:49:52.967Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.215356451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150702024-02-27T21:49:52.967Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.215400166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150712024-02-27T21:49:52.967Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.215438541Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150722024-02-27T21:49:52.967Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.215472677Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150732024-02-27T21:49:52.967Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.215513183Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150742024-02-27T21:49:52.967Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.215729727Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150752024-02-27T21:49:52.967Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.21613027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150762024-02-27T21:49:52.967Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216163816Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150772024-02-27T21:49:52.967Z{"msg":"[0]R flush_numbers[0..12]: [255, 255, 232, 244, 253, 256, 256, 256, 256, 252, 253, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216180614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150782024-02-27T21:49:52.967Z{"msg":"[0]R generation[0..12]: [138, 138, 124, 130, 137, 139, 139, 139, 139, 136, 137, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216208891Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150792024-02-27T21:49:52.967Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216223729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150802024-02-27T21:49:52.967Z{"msg":"[1]R flush_numbers[0..12]: [255, 255, 232, 244, 253, 256, 256, 256, 256, 252, 253, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216245706Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150812024-02-27T21:49:52.967Z{"msg":"[1]R generation[0..12]: [138, 138, 124, 130, 137, 139, 139, 139, 139, 136, 137, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216260025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150822024-02-27T21:49:52.968Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216287102Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150832024-02-27T21:49:52.968Z{"msg":"[2]R flush_numbers[0..12]: [255, 255, 232, 244, 253, 256, 256, 256, 256, 252, 253, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.21630194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150842024-02-27T21:49:52.968Z{"msg":"[2]R generation[0..12]: [138, 138, 124, 130, 137, 139, 139, 139, 139, 136, 137, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216316138Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150852024-02-27T21:49:52.968Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216337405Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150862024-02-27T21:49:52.968ZWait for a query_work_queue command to finish before sending IO
150872024-02-27T21:49:52.968Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216362153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150882024-02-27T21:49:52.968Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216376411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150892024-02-27T21:49:52.968Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216411817Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150902024-02-27T21:49:52.968Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216427045Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"":"downstairs","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150912024-02-27T21:49:52.968Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216440673Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150922024-02-27T21:49:52.968Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216460891Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"0","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150932024-02-27T21:49:52.968Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216474719Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"1","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150942024-02-27T21:49:52.968Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216494857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"client":"2","":"downstairs","session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150952024-02-27T21:49:52.968Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216509065Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150962024-02-27T21:49:52.968Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216521914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150972024-02-27T21:49:52.968Z{"msg":"85be349b-bcaf-4fa1-909e-f6ac46ddd87b is now active with session: 98be17fd-938f-488f-8e6b-455657d2e753","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216542471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150982024-02-27T21:49:52.968Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.21655479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707,"session_id":"98be17fd-938f-488f-8e6b-455657d2e753"}
150992024-02-27T21:49:52.968Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:51.216574108Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707}
151002024-02-27T21:49:52.968ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
151012024-02-27T21:49:52.968ZLoading write count information from file "/tmp/test_repair_verify.data"
151022024-02-27T21:49:52.968ZRun Repair workload
151032024-02-27T21:49:52.968Z01/30 Write block 154 len 16384 data: 8 11 13 13
151042024-02-27T21:49:52.968Z02/30 Read block 529 len 32768
151052024-02-27T21:49:52.968Z03/30 Read block 551 len 40960
151062024-02-27T21:49:52.968Z04/30 Read block 207 len 32768
151072024-02-27T21:49:52.974Z05/30 Read block 72 len 32768
151082024-02-27T21:49:52.974Z06/30 Write block 134 len 20480 data: 5 7 7 8 8
151092024-02-27T21:49:52.974Z07/30 Write block 227 len 40960 data: 10 11 10 13 14 17 16 17 17 17
151102024-02-27T21:49:52.974Z08/30 Write block 192 len 40960 data: 13 11 11 9 7 7 6 8 9 8
151112024-02-27T21:49:52.974Z09/30 Flush
151122024-02-27T21:49:52.974Z10/30 Write block 270 len 24576 data: 10 11 12 8 9 7
151132024-02-27T21:49:52.974Z11/30 Read block 68 len 20480
151142024-02-27T21:49:52.974Z12/30 Read block 576 len 12288
151152024-02-27T21:49:52.974Z13/30 Read block 493 len 28672
151162024-02-27T21:49:52.974Z14/30 Read block 520 len 16384
151172024-02-27T21:49:52.974Z15/30 Write block 300 len 28672 data: 11 12 12 10 13 11 12
151182024-02-27T21:49:52.974Z16/30 Read block 127 len 36864
151192024-02-27T21:49:52.975Z17/30 Write block 379 len 28672 data: 7 8 8 11 11 12 12
151202024-02-27T21:49:52.975Z18/30 Write block 167 len 4096 data: 5
151212024-02-27T21:49:52.975Z19/30 Read block 208 len 16384
151222024-02-27T21:49:52.975Z20/30 Read block 147 len 20480
151232024-02-27T21:49:52.975Z21/30 Read block 559 len 8192
151242024-02-27T21:49:52.975Z22/30 Write block 491 len 8192 data: 13 11
151252024-02-27T21:49:52.975Z23/30 Read block 161 len 4096
151262024-02-27T21:49:52.975Z24/30 Read block 437 len 36864
151272024-02-27T21:49:52.975Z25/30 Write block 111 len 16384 data: 14 12 9 7
151282024-02-27T21:49:52.975Z26/30 Write block 459 len 24576 data: 9 9 9 9 8 8
151292024-02-27T21:49:52.975Z27/30 Read block 562 len 28672
151302024-02-27T21:49:52.975Z28/30 Read block 277 len 4096
151312024-02-27T21:49:52.975Z29/30 Write block 165 len 12288 data: 9 6 6
151322024-02-27T21:49:52.975Z30/30 Read block 469 len 4096
151332024-02-27T21:49:52.975Z----------------------------------------------------------------
151342024-02-27T21:49:52.975Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
151352024-02-27T21:49:52.975ZStates: Active Active Active
151362024-02-27T21:49:52.975ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151372024-02-27T21:49:52.975Z 1 Acked 1000 Write 4 Done Done Done false
151382024-02-27T21:49:52.975Z 2 Acked 1001 Read 8 Done Done Done false
151392024-02-27T21:49:52.975Z 3 Acked 1002 Read 10 Done Done Sent false
151402024-02-27T21:49:52.975Z 4 Acked 1003 Read 8 Done Done Sent false
151412024-02-27T21:49:52.975Z 5 Acked 1004 Read 8 Done Done Sent false
151422024-02-27T21:49:52.975Z 6 Acked 1005 Write 5 Done Done Sent false
151432024-02-27T21:49:52.975Z 7 Acked 1006 Write 10 Done Done Sent false
151442024-02-27T21:49:52.976Z 8 Acked 1007 Write 10 Done Done Sent false
151452024-02-27T21:49:52.976Z 9 Acked 1008 Flush 0 Done Done Sent false
151462024-02-27T21:49:52.976Z 10 Acked 1009 Write 6 Done Done Sent false
151472024-02-27T21:49:52.976Z 11 Acked 1010 Read 5 Done Done Sent false
151482024-02-27T21:49:52.976Z 12 Acked 1011 Read 3 Done Done Sent false
151492024-02-27T21:49:52.976Z 13 Acked 1012 Read 7 Done Done Sent false
151502024-02-27T21:49:52.976Z 14 Acked 1013 Read 4 Done Done Sent false
151512024-02-27T21:49:52.976Z 15 Acked 1014 Write 7 Done Done Sent false
151522024-02-27T21:49:52.976Z 16 Acked 1015 Read 9 Done Done Sent false
151532024-02-27T21:49:52.976Z 17 Acked 1016 Write 7 Done Done Sent false
151542024-02-27T21:49:52.976Z 18 Acked 1017 Write 1 Done Done Sent false
151552024-02-27T21:49:52.976Z 19 Acked 1018 Read 4 Done Done Sent false
151562024-02-27T21:49:52.976Z 20 Acked 1019 Read 5 Done Done Sent false
151572024-02-27T21:49:52.976Z 21 Acked 1020 Read 2 Done Done Sent false
151582024-02-27T21:49:52.976Z 22 Acked 1021 Write 2 Done Done Sent false
151592024-02-27T21:49:52.976Z 23 Acked 1022 Read 1 Done Done Sent false
151602024-02-27T21:49:52.976Z 24 Acked 1023 Read 9 Done Done Sent false
151612024-02-27T21:49:52.976Z 25 Acked 1024 Write 4 Done Done Sent false
151622024-02-27T21:49:52.976Z 26 Acked 1025 Write 6 Done Done Sent false
151632024-02-27T21:49:52.976Z 27 Acked 1026 Read 7 Done Done Sent false
151642024-02-27T21:49:52.976Z 28 Acked 1027 Read 1 Done Done Sent false
151652024-02-27T21:49:52.976Z 29 Acked 1028 Write 3 Done Done Sent false
151662024-02-27T21:49:52.976Z 30 Acked 1029 Read 1 Done Done Sent false
151672024-02-27T21:49:52.976Z STATES DS:0 DS:1 DS:2 TOTAL
151682024-02-27T21:49:52.976Z New 0 0 0 0
151692024-02-27T21:49:52.977Z Sent 0 0 28 28
151702024-02-27T21:49:52.977Z Done 30 30 2 62
151712024-02-27T21:49:52.977Z Skipped 0 0 0 0
151722024-02-27T21:49:52.977Z Error 0 0 0 0
151732024-02-27T21:49:52.977ZLast Flush: 1008 1008 0
151742024-02-27T21:49:52.977ZDownstairs last five completed:
151752024-02-27T21:49:52.977ZUpstairs last five completed: 30 29 28 27 26
151762024-02-27T21:49:52.977Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:51.245530227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1707}
151772024-02-27T21:49:52.977ZWrote out file "/tmp/test_repair_verify.data"
151782024-02-27T21:49:52.977Z
151792024-02-27T21:49:52.977ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151802024-02-27T21:49:52.977ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151812024-02-27T21:49:52.977Z 0 000-019 138 138 138 255 255 255 F F F
151822024-02-27T21:49:52.977Z 1 020-039 138 138 138 255 255 255 F F F
151832024-02-27T21:49:52.977Z 2 040-059 124 124 124 232 232 232 F F F
151842024-02-27T21:49:52.977Z 3 060-079 130 130 130 244 244 244 F F F
151852024-02-27T21:49:52.977Z 4 080-099 137 137 137 253 253 253 F F F
151862024-02-27T21:49:52.977Z 5 100-119 139 139 139 256 256 256 T T F <---
151872024-02-27T21:49:52.977Z 6 120-139 140 140 139 257 257 256 F F F <---
151882024-02-27T21:49:52.977Z 7 140-159 140 140 139 257 257 256 F F T <---
151892024-02-27T21:49:52.977Z 8 160-179 139 139 139 256 256 256 T T F <---
151902024-02-27T21:49:52.977Z 9 180-199 140 140 136 257 257 252 F F F <---
151912024-02-27T21:49:52.977Z 10 200-219 140 140 137 257 257 253 F F F <---
151922024-02-27T21:49:52.977Z 11 220-239 140 140 132 257 257 247 F F F <---
151932024-02-27T21:49:52.977Z 12 240-259 136 136 136 252 252 252 F F F
151942024-02-27T21:49:52.978Z 13 260-279 139 139 139 256 256 256 T T F <---
151952024-02-27T21:49:52.978Z 14 280-299 137 137 137 253 253 253 F F F
151962024-02-27T21:49:52.978Z 15 300-319 136 136 136 252 252 252 T T F <---
151972024-02-27T21:49:52.978Z 16 320-339 139 139 139 256 256 256 F F F
151982024-02-27T21:49:52.978Z 17 340-359 139 139 139 256 256 256 F F F
151992024-02-27T21:49:52.978Z 18 360-379 138 138 138 254 254 254 T T F <---
152002024-02-27T21:49:52.978Z 19 380-399 138 138 138 254 254 254 T T F <---
152012024-02-27T21:49:52.978Z 20 400-419 136 136 136 252 252 252 F F F
152022024-02-27T21:49:52.978Z 21 420-439 136 136 136 252 252 252 F F F
152032024-02-27T21:49:52.978Z 22 440-459 139 139 139 256 256 256 T T F <---
152042024-02-27T21:49:52.978Z 23 460-479 136 136 136 252 252 252 T T F <---
152052024-02-27T21:49:52.978Z 24 480-499 135 135 135 251 251 251 T T F <---
152062024-02-27T21:49:52.978Z 25 500-519 138 138 138 254 254 254 F F F
152072024-02-27T21:49:52.982Z 26 520-539 135 135 135 251 251 251 F F F
152082024-02-27T21:49:52.982Z 27 540-559 138 138 138 254 254 254 F F F
152092024-02-27T21:49:52.982Z 28 560-579 137 137 137 253 253 253 F F F
152102024-02-27T21:49:52.982Z 29 580-599 138 138 138 254 254 254 F F F
152112024-02-27T21:49:52.982ZMax gen: 140, Max flush: 257
152122024-02-27T21:49:52.982ZError: Difference in extent metadata found!
152132024-02-27T21:49:52.982ZA Difference in extent metadata is expected here
152142024-02-27T21:49:52.982ZOn loop 70
152152024-02-27T21:49:52.982Z
152162024-02-27T21:49:52.982ZVerifying data now
152172024-02-27T21:49:58.608ZNo change after verify
152182024-02-27T21:49:58.608ZLoop: 70 Downstairs dump after verify (and repair):
152192024-02-27T21:49:58.622ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152202024-02-27T21:49:58.622Z 0 000-019 138 138 138 255 255 255 F F F
152212024-02-27T21:49:58.622Z 1 020-039 138 138 138 255 255 255 F F F
152222024-02-27T21:49:58.622Z 2 040-059 124 124 124 232 232 232 F F F
152232024-02-27T21:49:58.622Z 3 060-079 130 130 130 244 244 244 F F F
152242024-02-27T21:49:58.622Z 4 080-099 137 137 137 253 253 253 F F F
152252024-02-27T21:49:58.622Z 5 100-119 141 141 141 258 258 258 F F F
152262024-02-27T21:49:58.622Z 6 120-139 140 140 140 257 257 257 F F F
152272024-02-27T21:49:58.622Z 7 140-159 140 140 140 257 257 257 F F F
152282024-02-27T21:49:58.626Z 8 160-179 141 141 141 258 258 258 F F F
152292024-02-27T21:49:58.626Z 9 180-199 140 140 140 257 257 257 F F F
152302024-02-27T21:49:58.626Z 10 200-219 140 140 140 257 257 257 F F F
152312024-02-27T21:49:58.626Z 11 220-239 140 140 140 257 257 257 F F F
152322024-02-27T21:49:58.626Z 12 240-259 136 136 136 252 252 252 F F F
152332024-02-27T21:49:58.626Z 13 260-279 141 141 141 258 258 258 F F F
152342024-02-27T21:49:58.626Z 14 280-299 137 137 137 253 253 253 F F F
152352024-02-27T21:49:58.626Z 15 300-319 141 141 141 258 258 258 F F F
152362024-02-27T21:49:58.626Z 16 320-339 139 139 139 256 256 256 F F F
152372024-02-27T21:49:58.626Z 17 340-359 139 139 139 256 256 256 F F F
152382024-02-27T21:49:58.626Z 18 360-379 141 141 141 258 258 258 F F F
152392024-02-27T21:49:58.626Z 19 380-399 141 141 141 258 258 258 F F F
152402024-02-27T21:49:58.626Z 20 400-419 136 136 136 252 252 252 F F F
152412024-02-27T21:49:58.626Z 21 420-439 136 136 136 252 252 252 F F F
152422024-02-27T21:49:58.626Z 22 440-459 141 141 141 258 258 258 F F F
152432024-02-27T21:49:58.626Z 23 460-479 141 141 141 258 258 258 F F F
152442024-02-27T21:49:58.626Z 24 480-499 141 141 141 258 258 258 F F F
152452024-02-27T21:49:58.626Z 25 500-519 138 138 138 254 254 254 F F F
152462024-02-27T21:49:58.626Z 26 520-539 135 135 135 251 251 251 F F F
152472024-02-27T21:49:58.626Z 27 540-559 138 138 138 254 254 254 F F F
152482024-02-27T21:49:58.626Z 28 560-579 137 137 137 253 253 253 F F F
152492024-02-27T21:49:58.626Z 29 580-599 138 138 138 254 254 254 F F F
152502024-02-27T21:49:58.627ZMax gen: 141, Max flush: 258
152512024-02-27T21:49:58.627Z
152522024-02-27T21:49:58.627ZBegin loop 71
152532024-02-27T21:49:58.627ZDownstairs to restart: 1
152542024-02-27T21:49:58.644Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.013142385Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715}
152552024-02-27T21:49:58.644Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.013426562Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715}
152562024-02-27T21:49:58.649Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
152572024-02-27T21:49:58.649Z,"time":"2024-02-27T21:49:57.013449439Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715}
152582024-02-27T21:49:58.649Z{"msg":"Crucible 4d0786b7-1f38-44be-86b3-4f2a30660aa5 has session id: 168da868-ec56-4dd3-aad0-5230fde531d7","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.013543878Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152592024-02-27T21:49:58.650Z{"msg":"Upstairs opts: Upstairs UUID: 4d0786b7-1f38-44be-86b3-4f2a30660aa5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.01361156Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152602024-02-27T21:49:58.650Z{"msg":"Crucible stats registered with UUID: 4d0786b7-1f38-44be-86b3-4f2a30660aa5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.013657205Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152612024-02-27T21:49:58.650Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.0137008Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715}
152622024-02-27T21:49:58.650Z{"msg":"4d0786b7-1f38-44be-86b3-4f2a30660aa5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.013764582Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152632024-02-27T21:49:58.650Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.013811217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152642024-02-27T21:49:58.650Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.01386406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152652024-02-27T21:49:58.650Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.013908785Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152662024-02-27T21:49:58.650Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.01395344Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152672024-02-27T21:49:58.650Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.01403569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152682024-02-27T21:49:58.650Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014091404Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152692024-02-27T21:49:58.650Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014137118Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152702024-02-27T21:49:58.650Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014168175Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152712024-02-27T21:49:58.650Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.01421099Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152722024-02-27T21:49:58.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014252485Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152732024-02-27T21:49:58.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.01429162Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152742024-02-27T21:49:58.651Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:57.014326406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152752024-02-27T21:49:58.651Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:57.014369211Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152762024-02-27T21:49:58.651Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:57.014400947Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152772024-02-27T21:49:58.651Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014437833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152782024-02-27T21:49:58.651Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014478568Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152792024-02-27T21:49:58.651Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014508384Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152802024-02-27T21:49:58.651Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014770614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152812024-02-27T21:49:58.651Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014805529Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152822024-02-27T21:49:58.651Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014821918Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152832024-02-27T21:49:58.651Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.014855284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152842024-02-27T21:49:58.651Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.015353055Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152852024-02-27T21:49:58.651Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.01539559Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152862024-02-27T21:49:58.651Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.015890622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152872024-02-27T21:49:58.651Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.015927427Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152882024-02-27T21:49:58.651Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.015942576Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152892024-02-27T21:49:58.651Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.015962763Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152902024-02-27T21:49:58.652Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.01598721Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152912024-02-27T21:49:58.652Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:57.016004508Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152922024-02-27T21:49:59.646Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.01580354Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152932024-02-27T21:49:59.646Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.015918706Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152942024-02-27T21:49:59.652Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.016795443Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152952024-02-27T21:49:59.652Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.016833589Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152962024-02-27T21:49:59.652Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017312672Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152972024-02-27T21:49:59.652Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017793945Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152982024-02-27T21:49:59.652Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017823412Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
152992024-02-27T21:49:59.652Z{"msg":"[0]R flush_numbers[0..12]: [255, 255, 232, 244, 253, 258, 257, 257, 258, 257, 257, 257]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.01784487Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153002024-02-27T21:49:59.652Z{"msg":"[0]R generation[0..12]: [138, 138, 124, 130, 137, 141, 140, 140, 141, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017861217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153012024-02-27T21:49:59.652Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017884085Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153022024-02-27T21:49:59.652Z{"msg":"[1]R flush_numbers[0..12]: [255, 255, 232, 244, 253, 258, 257, 257, 258, 257, 257, 257]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017899433Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153032024-02-27T21:49:59.652Z{"msg":"[1]R generation[0..12]: [138, 138, 124, 130, 137, 141, 140, 140, 141, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017921181Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153042024-02-27T21:49:59.652Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017936229Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153052024-02-27T21:49:59.652Z{"msg":"[2]R flush_numbers[0..12]: [255, 255, 232, 244, 253, 258, 257, 257, 258, 257, 257, 257]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017957436Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153062024-02-27T21:49:59.652Z{"msg":"[2]R generation[0..12]: [138, 138, 124, 130, 137, 141, 140, 140, 141, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017971775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153072024-02-27T21:49:59.652Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017986323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153082024-02-27T21:49:59.652Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.017999881Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153092024-02-27T21:49:59.652Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018021239Z","hostname":"Wait for a query_work_queue command to finish before sending IO
153102024-02-27T21:49:59.653Zip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153112024-02-27T21:49:59.653Z{"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018045666Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153122024-02-27T21:49:59.653Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018068503Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"":"downstairs","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153132024-02-27T21:49:59.653Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018091201Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153142024-02-27T21:49:59.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018103909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"0","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153152024-02-27T21:49:59.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018118797Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"1","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153162024-02-27T21:49:59.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018132736Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"client":"2","":"downstairs","session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153172024-02-27T21:49:59.653Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018154373Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153182024-02-27T21:49:59.653Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018173341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153192024-02-27T21:49:59.653Z{"msg":"4d0786b7-1f38-44be-86b3-4f2a30660aa5 is now active with session: 168da868-ec56-4dd3-aad0-5230fde531d7","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018188439Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153202024-02-27T21:49:59.653Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018208067Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715,"session_id":"168da868-ec56-4dd3-aad0-5230fde531d7"}
153212024-02-27T21:49:59.653Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:49:58.018224585Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715}
153222024-02-27T21:49:59.653ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153232024-02-27T21:49:59.653ZLoading write count information from file "/tmp/test_repair_verify.data"
153242024-02-27T21:49:59.653ZRun Repair workload
153252024-02-27T21:49:59.656Z01/30 Write block 136 len 20480 data: 8 9 9 10 10
153262024-02-27T21:49:59.657Z02/30 Write block 519 len 28672 data: 9 12 9 10 8 9 8
153272024-02-27T21:49:59.657Z03/30 Read block 50 len 32768
153282024-02-27T21:49:59.657Z04/30 Write block 113 len 40960 data: 10 8 8 7 9 11 12 14 16 14
153292024-02-27T21:49:59.657Z05/30 Read block 417 len 36864
153302024-02-27T21:49:59.657Z06/30 Write block 287 len 36864 data: 8 8 11 12 12 11 14 14 15
153312024-02-27T21:49:59.660Z07/30 Write block 517 len 8192 data: 13 10
153322024-02-27T21:49:59.660Z08/30 Write block 318 len 20480 data: 14 16 16 15 13
153332024-02-27T21:49:59.660Z09/30 Write block 567 len 4096 data: 7
153342024-02-27T21:49:59.660Z10/30 Read block 174 len 12288
153352024-02-27T21:49:59.660Z11/30 Write block 537 len 36864 data: 8 6 6 7 7 8 8 7 8
153362024-02-27T21:49:59.663Z12/30 Write block 355 len 20480 data: 15 14 9 9 10
153372024-02-27T21:49:59.663Z13/30 Read block 517 len 20480
153382024-02-27T21:49:59.663Z14/30 Write block 241 len 4096 data: 8
153392024-02-27T21:49:59.663Z15/30 Read block 317 len 4096
153402024-02-27T21:49:59.663Z16/30 Write block 184 len 40960 data: 16 14 15 14 17 15 13 15 14 12
153412024-02-27T21:49:59.667Z17/30 Read block 253 len 16384
153422024-02-27T21:49:59.667Z18/30 Write block 61 len 12288 data: 6 7 9
153432024-02-27T21:49:59.667Z19/30 Read block 450 len 40960
153442024-02-27T21:49:59.667Z20/30 Flush
153452024-02-27T21:49:59.671Z21/30 Write block 171 len 8192 data: 7 7
153462024-02-27T21:49:59.675Z22/30 Read block 115 len 40960
153472024-02-27T21:49:59.675Z23/30 Read block 77 len 36864
153482024-02-27T21:49:59.675Z24/30 Read block 353 len 40960
153492024-02-27T21:49:59.678Z25/30 Read block 301 len 4096
153502024-02-27T21:49:59.678Z26/30 Read block 185 len 12288
153512024-02-27T21:49:59.678Z27/30 Read block 65 len 24576
153522024-02-27T21:49:59.678Z28/30 Read block 54 len 36864
153532024-02-27T21:49:59.681Z29/30 Read block 199 len 20480
153542024-02-27T21:49:59.681Z30/30 Read block 305 len 12288
153552024-02-27T21:49:59.682Z----------------------------------------------------------------
153562024-02-27T21:49:59.682Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
153572024-02-27T21:49:59.682ZStates: Active Active Active
153582024-02-27T21:49:59.682ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153592024-02-27T21:49:59.682Z 1 Acked 1000 Write 5 Done Done Done false
153602024-02-27T21:49:59.682Z 2 Acked 1001 Write 7 Done Sent Done false
153612024-02-27T21:49:59.682Z 3 Acked 1002 Read 8 Done Sent Done false
153622024-02-27T21:49:59.682Z 4 Acked 1003 Write 10 Done Sent Done false
153632024-02-27T21:49:59.682Z 5 Acked 1004 Read 9 Done Sent Done false
153642024-02-27T21:49:59.682Z 6 Acked 1005 Write 9 Done Sent Done false
153652024-02-27T21:49:59.682Z 7 Acked 1006 Write 2 Done Sent Done false
153662024-02-27T21:49:59.682Z 8 Acked 1007 Write 5 Done Sent Done false
153672024-02-27T21:49:59.682Z 9 Acked 1008 Write 1 Done Sent Done false
153682024-02-27T21:49:59.682Z 10 Acked 1009 Read 3 Done Sent Done false
153692024-02-27T21:49:59.682Z 11 Acked 1010 Write 9 Done Sent Done false
153702024-02-27T21:49:59.682Z 12 Acked 1011 Write 5 Done Sent Done false
153712024-02-27T21:49:59.682Z 13 Acked 1012 Read 5 Done Sent Done false
153722024-02-27T21:49:59.682Z 14 Acked 1013 Write 1 Done Sent Done false
153732024-02-27T21:49:59.682Z 15 Acked 1014 Read 1 Done Sent Done false
153742024-02-27T21:49:59.682Z 16 Acked 1015 Write 10 Done Sent Done false
153752024-02-27T21:49:59.682Z 17 Acked 1016 Read 4 Done Sent Done false
153762024-02-27T21:49:59.682Z 18 Acked 1017 Write 3 Done Sent Done false
153772024-02-27T21:49:59.682Z 19 Acked 1018 Read 10 Done Sent Done false
153782024-02-27T21:49:59.682Z 20 Acked 1019 Flush 0 Done Sent Done false
153792024-02-27T21:49:59.683Z 21 Acked 1020 Write 2 Done Sent Done false
153802024-02-27T21:49:59.683Z 22 Acked 1021 Read 10 Done Sent Done false
153812024-02-27T21:49:59.683Z 23 Acked 1022 Read 9 Done Sent Done false
153822024-02-27T21:49:59.683Z 24 Acked 1023 Read 10 Done Sent Done false
153832024-02-27T21:49:59.683Z 25 Acked 1024 Read 1 Done Sent Done false
153842024-02-27T21:49:59.683Z 26 Acked 1025 Read 3 Done Sent Done false
153852024-02-27T21:49:59.683Z 27 Acked 1026 Read 6 Done Sent Done false
153862024-02-27T21:49:59.683Z 28 Acked 1027 Read 9 Done Sent Done false
153872024-02-27T21:49:59.683Z 29 Acked 1028 Read 5 Done Sent Done false
153882024-02-27T21:49:59.683Z 30 Acked 1029 Read 3 Done Sent Done false
153892024-02-27T21:49:59.683Z STATES DS:0 DS:1 DS:2 TOTAL
153902024-02-27T21:49:59.683Z New 0 0 0 0
153912024-02-27T21:49:59.683Z Sent 0 29 0 29
153922024-02-27T21:49:59.683Z Done 30 1 30 61
153932024-02-27T21:49:59.683Z Skipped 0 0 0 0
153942024-02-27T21:49:59.683Z Error 0 0 0 0
153952024-02-27T21:49:59.683ZLast Flush: 1019 0 1019
153962024-02-27T21:49:59.683ZDownstairs last five completed:
153972024-02-27T21:49:59.683ZUpstairs last five completed: 30 29 28 27 26
153982024-02-27T21:49:59.683Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:49:58.049579521Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1715}
153992024-02-27T21:49:59.683ZWrote out file "/tmp/test_repair_verify.data"
154002024-02-27T21:49:59.687Z
154012024-02-27T21:49:59.690ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
154022024-02-27T21:49:59.702ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154032024-02-27T21:49:59.703Z 0 000-019 138 138 138 255 255 255 F F F
154042024-02-27T21:49:59.703Z 1 020-039 138 138 138 255 255 255 F F F
154052024-02-27T21:49:59.703Z 2 040-059 124 124 124 232 232 232 F F F
154062024-02-27T21:49:59.703Z 3 060-079 142 130 142 259 244 259 F F F <---
154072024-02-27T21:49:59.703Z 4 080-099 137 137 137 253 253 253 F F F
154082024-02-27T21:49:59.706Z 5 100-119 142 141 142 259 258 259 F F F <---
154092024-02-27T21:49:59.706Z 6 120-139 142 140 142 259 257 259 F T F <---
154102024-02-27T21:49:59.706Z 7 140-159 142 140 142 259 257 259 F T F <---
154112024-02-27T21:49:59.706Z 8 160-179 141 141 141 258 258 258 T F T <---
154122024-02-27T21:49:59.706Z 9 180-199 142 140 142 259 257 259 F F F <---
154132024-02-27T21:49:59.706Z 10 200-219 140 140 140 257 257 257 F F F
154142024-02-27T21:49:59.706Z 11 220-239 140 140 140 257 257 257 F F F
154152024-02-27T21:49:59.706Z 12 240-259 142 136 142 259 252 259 F F F <---
154162024-02-27T21:49:59.706Z 13 260-279 141 141 141 258 258 258 F F F
154172024-02-27T21:49:59.706Z 14 280-299 142 137 142 259 253 259 F F F <---
154182024-02-27T21:49:59.706Z 15 300-319 142 141 142 259 258 259 F F F <---
154192024-02-27T21:49:59.706Z 16 320-339 142 139 142 259 256 259 F F F <---
154202024-02-27T21:49:59.707Z 17 340-359 142 139 142 259 256 259 F F F <---
154212024-02-27T21:49:59.707Z 18 360-379 141 141 141 258 258 258 F F F
154222024-02-27T21:49:59.707Z 19 380-399 141 141 141 258 258 258 F F F
154232024-02-27T21:49:59.707Z 20 400-419 136 136 136 252 252 252 F F F
154242024-02-27T21:49:59.707Z 21 420-439 136 136 136 252 252 252 F F F
154252024-02-27T21:49:59.707Z 22 440-459 141 141 141 258 258 258 F F F
154262024-02-27T21:49:59.707Z 23 460-479 141 141 141 258 258 258 F F F
154272024-02-27T21:49:59.707Z 24 480-499 141 141 141 258 258 258 F F F
154282024-02-27T21:49:59.707Z 25 500-519 142 138 142 259 254 259 F F F <---
154292024-02-27T21:49:59.707Z 26 520-539 142 135 142 259 251 259 F F F <---
154302024-02-27T21:49:59.707Z 27 540-559 142 138 142 259 254 259 F F F <---
154312024-02-27T21:49:59.707Z 28 560-579 142 137 142 259 253 259 F F F <---
154322024-02-27T21:49:59.707Z 29 580-599 138 138 138 254 254 254 F F F
154332024-02-27T21:49:59.707ZMax gen: 142, Max flush: 259
154342024-02-27T21:49:59.707ZError: Difference in extent metadata found!
154352024-02-27T21:49:59.707ZA Difference in extent metadata is expected here
154362024-02-27T21:49:59.707ZOn loop 71
154372024-02-27T21:49:59.707Z
154382024-02-27T21:49:59.713ZVerifying data now
154392024-02-27T21:50:05.447ZNo change after verify
154402024-02-27T21:50:05.447ZLoop: 71 Downstairs dump after verify (and repair):
154412024-02-27T21:50:05.462ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154422024-02-27T21:50:05.462Z 0 000-019 138 138 138 255 255 255 F F F
154432024-02-27T21:50:05.462Z 1 020-039 138 138 138 255 255 255 F F F
154442024-02-27T21:50:05.462Z 2 040-059 124 124 124 232 232 232 F F F
154452024-02-27T21:50:05.462Z 3 060-079 142 142 142 259 259 259 F F F
154462024-02-27T21:50:05.462Z 4 080-099 137 137 137 253 253 253 F F F
154472024-02-27T21:50:05.462Z 5 100-119 142 142 142 259 259 259 F F F
154482024-02-27T21:50:05.462Z 6 120-139 142 142 142 259 259 259 F F F
154492024-02-27T21:50:05.462Z 7 140-159 142 142 142 259 259 259 F F F
154502024-02-27T21:50:05.462Z 8 160-179 143 143 143 260 260 260 F F F
154512024-02-27T21:50:05.462Z 9 180-199 142 142 142 259 259 259 F F F
154522024-02-27T21:50:05.462Z 10 200-219 140 140 140 257 257 257 F F F
154532024-02-27T21:50:05.466Z 11 220-239 140 140 140 257 257 257 F F F
154542024-02-27T21:50:05.466Z 12 240-259 142 142 142 259 259 259 F F F
154552024-02-27T21:50:05.466Z 13 260-279 141 141 141 258 258 258 F F F
154562024-02-27T21:50:05.466Z 14 280-299 142 142 142 259 259 259 F F F
154572024-02-27T21:50:05.466Z 15 300-319 142 142 142 259 259 259 F F F
154582024-02-27T21:50:05.466Z 16 320-339 142 142 142 259 259 259 F F F
154592024-02-27T21:50:05.466Z 17 340-359 142 142 142 259 259 259 F F F
154602024-02-27T21:50:05.466Z 18 360-379 141 141 141 258 258 258 F F F
154612024-02-27T21:50:05.467Z 19 380-399 141 141 141 258 258 258 F F F
154622024-02-27T21:50:05.467Z 20 400-419 136 136 136 252 252 252 F F F
154632024-02-27T21:50:05.467Z 21 420-439 136 136 136 252 252 252 F F F
154642024-02-27T21:50:05.467Z 22 440-459 141 141 141 258 258 258 F F F
154652024-02-27T21:50:05.467Z 23 460-479 141 141 141 258 258 258 F F F
154662024-02-27T21:50:05.467Z 24 480-499 141 141 141 258 258 258 F F F
154672024-02-27T21:50:05.467Z 25 500-519 142 142 142 259 259 259 F F F
154682024-02-27T21:50:05.467Z 26 520-539 142 142 142 259 259 259 F F F
154692024-02-27T21:50:05.467Z 27 540-559 142 142 142 259 259 259 F F F
154702024-02-27T21:50:05.467Z 28 560-579 142 142 142 259 259 259 F F F
154712024-02-27T21:50:05.467Z 29 580-599 138 138 138 254 254 254 F F F
154722024-02-27T21:50:05.467ZMax gen: 143, Max flush: 260
154732024-02-27T21:50:05.467Z
154742024-02-27T21:50:05.467ZBegin loop 72
154752024-02-27T21:50:05.467ZDownstairs to restart: 1
154762024-02-27T21:50:05.483Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.851351027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723}
154772024-02-27T21:50:05.483Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
154782024-02-27T21:50:05.491Z,"v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.851717464Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723}
154792024-02-27T21:50:05.492Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.851766748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723}
154802024-02-27T21:50:05.492Z{"msg":"Crucible 796bff61-46e4-48da-91ec-8076c701c739 has session id: 461f0337-84d8-4192-941b-1e093ca879ac","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.851819012Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154812024-02-27T21:50:05.492Z{"msg":"Upstairs opts: Upstairs UUID: 796bff61-46e4-48da-91ec-8076c701c739, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.851873226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154822024-02-27T21:50:05.492Z{"msg":"Crucible stats registered with UUID: 796bff61-46e4-48da-91ec-8076c701c739","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.85191851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154832024-02-27T21:50:05.492Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.851963695Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723}
154842024-02-27T21:50:05.492Z{"msg":"796bff61-46e4-48da-91ec-8076c701c739 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.85208985Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154852024-02-27T21:50:05.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852144124Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154862024-02-27T21:50:05.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852183049Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154872024-02-27T21:50:05.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852227194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154882024-02-27T21:50:05.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852274208Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154892024-02-27T21:50:05.492Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852307904Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154902024-02-27T21:50:05.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852349339Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154912024-02-27T21:50:05.492Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852379286Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154922024-02-27T21:50:05.492Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852419361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154932024-02-27T21:50:05.492Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852445318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154942024-02-27T21:50:05.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852476244Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154952024-02-27T21:50:05.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852677851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154962024-02-27T21:50:05.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.852738914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154972024-02-27T21:50:05.492Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.853144166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154982024-02-27T21:50:05.492Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.85319717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
154992024-02-27T21:50:05.492Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.853242124Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155002024-02-27T21:50:05.492Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.85327704Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155012024-02-27T21:50:05.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.853303447Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155022024-02-27T21:50:05.493Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.853345652Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155032024-02-27T21:50:05.493Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.853808168Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155042024-02-27T21:50:05.493Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.853846803Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155052024-02-27T21:50:05.493Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.853864501Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155062024-02-27T21:50:05.493Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854317958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155072024-02-27T21:50:05.493Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854370512Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155082024-02-27T21:50:05.493Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854419706Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155092024-02-27T21:50:05.493Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854450502Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155102024-02-27T21:50:05.493Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854481758Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155112024-02-27T21:50:05.493Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854506755Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155122024-02-27T21:50:05.493Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854534242Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155132024-02-27T21:50:05.493Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854558449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155142024-02-27T21:50:05.493Z{"msg":"[0]R flush_numbers[0..12]: [255, 255, 232, 259, 253, 259, 259, 259, 260, 259, 257, 257]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854586896Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155152024-02-27T21:50:05.493Z{"msg":"[0]R generation[0..12]: [138, 138, 124, 142, 137, 142, 142, 142, 143, 142, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854612483Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155162024-02-27T21:50:05.493ZWait for a query_work_queue command to finish before sending IO
155172024-02-27T21:50:05.493Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854663847Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155182024-02-27T21:50:05.493Z{"msg":"[1]R flush_numbers[0..12]: [255, 255, 232, 259, 253, 259, 259, 259, 260, 259, 257, 257]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854698703Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155192024-02-27T21:50:05.493Z{"msg":"[1]R generation[0..12]: [138, 138, 124, 142, 137, 142, 142, 142, 143, 142, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.85472521Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155202024-02-27T21:50:05.493Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854758676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155212024-02-27T21:50:05.493Z{"msg":"[2]R flush_numbers[0..12]: [255, 255, 232, 259, 253, 259, 259, 259, 260, 259, 257, 257]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854786792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155222024-02-27T21:50:05.493Z{"msg":"[2]R generation[0..12]: [138, 138, 124, 142, 137, 142, 142, 142, 143, 142, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854819099Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155232024-02-27T21:50:05.493Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854846005Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155242024-02-27T21:50:05.493Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854877452Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155252024-02-27T21:50:05.493Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854902379Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155262024-02-27T21:50:05.494Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854932665Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155272024-02-27T21:50:05.494Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854957182Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"":"downstairs","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155282024-02-27T21:50:05.494Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.854984019Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155292024-02-27T21:50:05.494Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.855005697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"0","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155302024-02-27T21:50:05.494Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.855032783Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"1","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155312024-02-27T21:50:05.494ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155322024-02-27T21:50:05.494Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.855070719Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"client":"2","":"downstairs","session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155332024-02-27T21:50:05.494Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.855095416Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155342024-02-27T21:50:05.494Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.855121483Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155352024-02-27T21:50:05.494Z{"msg":"796bff61-46e4-48da-91ec-8076c701c739 is now active with session: 461f0337-84d8-4192-941b-1e093ca879ac","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.85514484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155362024-02-27T21:50:05.494Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.855170077Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723,"session_id":"461f0337-84d8-4192-941b-1e093ca879ac"}
155372024-02-27T21:50:05.494Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:03.855192955Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723}
155382024-02-27T21:50:05.494ZLoading write count information from file "/tmp/test_repair_verify.data"
155392024-02-27T21:50:05.494ZRun Repair workload
155402024-02-27T21:50:05.494Z01/30 Write block 447 len 8192 data: 6 8
155412024-02-27T21:50:05.499Z02/30 Write block 167 len 4096 data: 7
155422024-02-27T21:50:05.499Z03/30 Read block 194 len 24576
155432024-02-27T21:50:05.499Z04/30 Flush
155442024-02-27T21:50:05.499Z05/30 Write block 344 len 32768 data: 14 12 12 12 16 17 16 14
155452024-02-27T21:50:05.499Z06/30 Write block 514 len 24576 data: 15 10 11 14 11 10
155462024-02-27T21:50:05.499Z07/30 Write block 249 len 28672 data: 12 12 12 12 12 11 15
155472024-02-27T21:50:05.499Z08/30 Flush
155482024-02-27T21:50:05.500Z09/30 Write block 417 len 32768 data: 18 16 15 10 7 8 9 9
155492024-02-27T21:50:05.503Z10/30 Read block 206 len 32768
155502024-02-27T21:50:05.503Z11/30 Flush
155512024-02-27T21:50:05.503Z12/30 Write block 288 len 8192 data: 9 12
155522024-02-27T21:50:05.503Z13/30 Write block 543 len 28672 data: 9 8 9 9 9 14 14
155532024-02-27T21:50:05.503Z14/30 Read block 260 len 28672
155542024-02-27T21:50:05.507Z15/30 Read block 590 len 32768
155552024-02-27T21:50:05.507Z16/30 Read block 144 len 32768
155562024-02-27T21:50:05.507Z17/30 Write block 215 len 20480 data: 12 13 15 15 13
155572024-02-27T21:50:05.507Z18/30 Write block 86 len 20480 data: 9 11 12 13 10
155582024-02-27T21:50:05.511Z19/30 Read block 340 len 4096
155592024-02-27T21:50:05.511Z20/30 Flush
155602024-02-27T21:50:05.514Z21/30 Write block 99 len 4096 data: 11
155612024-02-27T21:50:05.514Z22/30 Write block 48 len 32768 data: 11 11 12 12 13 12 11 7
155622024-02-27T21:50:05.514Z23/30 Write block 214 len 36864 data: 11 13 14 16 16 14 12 11 12
155632024-02-27T21:50:05.514Z24/30 Read block 225 len 24576
155642024-02-27T21:50:05.519Z25/30 Write block 59 len 16384 data: 10 8 7 8
155652024-02-27T21:50:05.519Z26/30 Read block 149 len 32768
155662024-02-27T21:50:05.519Z27/30 Write block 374 len 4096 data: 6
155672024-02-27T21:50:05.519Z28/30 Write block 346 len 28672 data: 13 13 17 18 17 15 11
155682024-02-27T21:50:05.519Z29/30 Read block 156 len 28672
155692024-02-27T21:50:05.523Z30/30 Read block 319 len 24576
155702024-02-27T21:50:05.523Z----------------------------------------------------------------
155712024-02-27T21:50:05.523Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
155722024-02-27T21:50:05.523ZStates: Active Active Active
155732024-02-27T21:50:05.523ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155742024-02-27T21:50:05.523Z 1 Acked 1000 Write 2 Done Done Done false
155752024-02-27T21:50:05.523Z 2 Acked 1001 Write 1 Done Done Done false
155762024-02-27T21:50:05.524Z 3 Acked 1002 Read 6 Done Sent Done false
155772024-02-27T21:50:05.524Z 4 Acked 1003 Flush 0 Done Sent Done false
155782024-02-27T21:50:05.524Z 5 Acked 1004 Write 8 Done Sent Done false
155792024-02-27T21:50:05.524Z 6 Acked 1005 Write 6 Done Sent Done false
155802024-02-27T21:50:05.524Z 7 Acked 1006 Write 7 Done Sent Done false
155812024-02-27T21:50:05.524Z 8 Acked 1007 Flush 0 Done Sent Done false
155822024-02-27T21:50:05.524Z 9 Acked 1008 Write 8 Done Sent Done false
155832024-02-27T21:50:05.524Z 10 Acked 1009 Read 8 Done Sent Done false
155842024-02-27T21:50:05.524Z 11 Acked 1010 Flush 0 Done Sent Done false
155852024-02-27T21:50:05.524Z 12 Acked 1011 Write 2 Done Sent Done false
155862024-02-27T21:50:05.524Z 13 Acked 1012 Write 7 Done Sent Done false
155872024-02-27T21:50:05.524Z 14 Acked 1013 Read 7 Done Sent Done false
155882024-02-27T21:50:05.524Z 15 Acked 1014 Read 8 Done Sent Done false
155892024-02-27T21:50:05.524Z 16 Acked 1015 Read 8 Done Sent Done false
155902024-02-27T21:50:05.524Z 17 Acked 1016 Write 5 Done Sent Done false
155912024-02-27T21:50:05.524Z 18 Acked 1017 Write 5 Done Sent Done false
155922024-02-27T21:50:05.524Z 19 Acked 1018 Read 1 Done Sent Done false
155932024-02-27T21:50:05.524Z 20 Acked 1019 Flush 0 Done Sent Done false
155942024-02-27T21:50:05.524Z 21 Acked 1020 Write 1 Done Sent Done false
155952024-02-27T21:50:05.524Z 22 Acked 1021 Write 8 Done Sent Done false
155962024-02-27T21:50:05.524Z 23 Acked 1022 Write 9 Done Sent Done false
155972024-02-27T21:50:05.524Z 24 Acked 1023 Read 6 Done Sent Done false
155982024-02-27T21:50:05.524Z 25 Acked 1024 Write 4 Done Sent Done false
155992024-02-27T21:50:05.525Z 26 Acked 1025 Read 8 Done Sent Done false
156002024-02-27T21:50:05.525Z 27 Acked 1026 Write 1 Done Sent Done false
156012024-02-27T21:50:05.525Z 28 Acked 1027 Write 7 Done Sent Done false
156022024-02-27T21:50:05.525Z 29 Acked 1028 Read 7 Done Sent Done false
156032024-02-27T21:50:05.525Z 30 Acked 1029 Read 6 Done Sent Done false
156042024-02-27T21:50:05.525Z STATES DS:0 DS:1 DS:2 TOTAL
156052024-02-27T21:50:05.525Z New 0 0 0 0
156062024-02-27T21:50:05.525Z Sent 0 28 0 28
156072024-02-27T21:50:05.525Z Done 30 2 30 62
156082024-02-27T21:50:05.525Z Skipped 0 0 0 0
156092024-02-27T21:50:05.525Z Error 0 0 0 0
156102024-02-27T21:50:05.525ZLast Flush: 1019 0 1019
156112024-02-27T21:50:05.525ZDownstairs last five completed:
156122024-02-27T21:50:05.525ZUpstairs last five completed: 30 29 28 27 26
156132024-02-27T21:50:05.525Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:03.887969064Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1723}
156142024-02-27T21:50:05.525ZWrote out file "/tmp/test_repair_verify.data"
156152024-02-27T21:50:05.529Z
156162024-02-27T21:50:05.529ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
156172024-02-27T21:50:05.545ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156182024-02-27T21:50:05.545Z 0 000-019 138 138 138 255 255 255 F F F
156192024-02-27T21:50:05.545Z 1 020-039 138 138 138 255 255 255 F F F
156202024-02-27T21:50:05.545Z 2 040-059 124 124 124 232 232 232 T F T <---
156212024-02-27T21:50:05.545Z 3 060-079 142 142 142 259 259 259 T F T <---
156222024-02-27T21:50:05.545Z 4 080-099 144 137 144 264 253 264 T F T <---
156232024-02-27T21:50:05.546Z 5 100-119 142 142 142 259 259 259 F F F
156242024-02-27T21:50:05.546Z 6 120-139 142 142 142 259 259 259 F F F
156252024-02-27T21:50:05.546Z 7 140-159 142 142 142 259 259 259 F F F
156262024-02-27T21:50:05.546Z 8 160-179 144 143 144 261 260 261 F T F <---
156272024-02-27T21:50:05.546Z 9 180-199 142 142 142 259 259 259 F F F
156282024-02-27T21:50:05.546Z 10 200-219 144 140 144 264 257 264 T F T <---
156292024-02-27T21:50:05.549Z 11 220-239 140 140 140 257 257 257 T F T <---
156302024-02-27T21:50:05.549Z 12 240-259 144 142 144 262 259 262 F F F <---
156312024-02-27T21:50:05.549Z 13 260-279 141 141 141 258 258 258 F F F
156322024-02-27T21:50:05.550Z 14 280-299 144 142 144 264 259 264 F F F <---
156332024-02-27T21:50:05.550Z 15 300-319 142 142 142 259 259 259 F F F
156342024-02-27T21:50:05.550Z 16 320-339 142 142 142 259 259 259 F F F
156352024-02-27T21:50:05.550Z 17 340-359 144 142 144 262 259 262 T F T <---
156362024-02-27T21:50:05.550Z 18 360-379 141 141 141 258 258 258 T F T <---
156372024-02-27T21:50:05.550Z 19 380-399 141 141 141 258 258 258 F F F
156382024-02-27T21:50:05.550Z 20 400-419 144 136 144 263 252 263 F F F <---
156392024-02-27T21:50:05.550Z 21 420-439 144 136 144 263 252 263 F F F <---
156402024-02-27T21:50:05.550Z 22 440-459 144 141 144 261 258 261 F T F <---
156412024-02-27T21:50:05.550Z 23 460-479 141 141 141 258 258 258 F F F
156422024-02-27T21:50:05.550Z 24 480-499 141 141 141 258 258 258 F F F
156432024-02-27T21:50:05.550Z 25 500-519 144 142 144 262 259 262 F F F <---
156442024-02-27T21:50:05.550Z 26 520-539 142 142 142 259 259 259 F F F
156452024-02-27T21:50:05.550Z 27 540-559 144 142 144 264 259 264 F F F <---
156462024-02-27T21:50:05.550Z 28 560-579 142 142 142 259 259 259 F F F
156472024-02-27T21:50:05.550Z 29 580-599 138 138 138 254 254 254 F F F
156482024-02-27T21:50:05.550ZMax gen: 144, Max flush: 264
156492024-02-27T21:50:05.550ZError: Difference in extent metadata found!
156502024-02-27T21:50:05.550ZA Difference in extent metadata is expected here
156512024-02-27T21:50:05.550ZOn loop 72
156522024-02-27T21:50:05.550Z
156532024-02-27T21:50:05.550ZVerifying data now
156542024-02-27T21:50:10.298ZNo change after verify
156552024-02-27T21:50:10.298ZLoop: 72 Downstairs dump after verify (and repair):
156562024-02-27T21:50:10.308ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156572024-02-27T21:50:10.308Z 0 000-019 138 138 138 255 255 255 F F F
156582024-02-27T21:50:10.322Z 1 020-039 138 138 138 255 255 255 F F F
156592024-02-27T21:50:10.322Z 2 040-059 145 145 145 265 265 265 F F F
156602024-02-27T21:50:10.322Z 3 060-079 145 145 145 265 265 265 F F F
156612024-02-27T21:50:10.322Z 4 080-099 145 145 145 265 265 265 F F F
156622024-02-27T21:50:10.322Z 5 100-119 142 142 142 259 259 259 F F F
156632024-02-27T21:50:10.322Z 6 120-139 142 142 142 259 259 259 F F F
156642024-02-27T21:50:10.322Z 7 140-159 142 142 142 259 259 259 F F F
156652024-02-27T21:50:10.322Z 8 160-179 144 144 144 261 261 261 F F F
156662024-02-27T21:50:10.322Z 9 180-199 142 142 142 259 259 259 F F F
156672024-02-27T21:50:10.322Z 10 200-219 145 145 145 265 265 265 F F F
156682024-02-27T21:50:10.322Z 11 220-239 145 145 145 265 265 265 F F F
156692024-02-27T21:50:10.323Z 12 240-259 144 144 144 262 262 262 F F F
156702024-02-27T21:50:10.323Z 13 260-279 141 141 141 258 258 258 F F F
156712024-02-27T21:50:10.323Z 14 280-299 144 144 144 264 264 264 F F F
156722024-02-27T21:50:10.323Z 15 300-319 142 142 142 259 259 259 F F F
156732024-02-27T21:50:10.323Z 16 320-339 142 142 142 259 259 259 F F F
156742024-02-27T21:50:10.323Z 17 340-359 145 145 145 265 265 265 F F F
156752024-02-27T21:50:10.323Z 18 360-379 145 145 145 265 265 265 F F F
156762024-02-27T21:50:10.323Z 19 380-399 141 141 141 258 258 258 F F F
156772024-02-27T21:50:10.323Z 20 400-419 144 144 144 263 263 263 F F F
156782024-02-27T21:50:10.323Z 21 420-439 144 144 144 263 263 263 F F F
156792024-02-27T21:50:10.323Z 22 440-459 144 144 144 261 261 261 F F F
156802024-02-27T21:50:10.323Z 23 460-479 141 141 141 258 258 258 F F F
156812024-02-27T21:50:10.323Z 24 480-499 141 141 141 258 258 258 F F F
156822024-02-27T21:50:10.323Z 25 500-519 144 144 144 262 262 262 F F F
156832024-02-27T21:50:10.323Z 26 520-539 142 142 142 259 259 259 F F F
156842024-02-27T21:50:10.323Z 27 540-559 144 144 144 264 264 264 F F F
156852024-02-27T21:50:10.323Z 28 560-579 142 142 142 259 259 259 F F F
156862024-02-27T21:50:10.323Z 29 580-599 138 138 138 254 254 254 F F F
156872024-02-27T21:50:10.323ZMax gen: 145, Max flush: 265
156882024-02-27T21:50:10.323Z
156892024-02-27T21:50:10.323ZBegin loop 73
156902024-02-27T21:50:10.323ZDownstairs to restart: 2
156912024-02-27T21:50:10.331Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698024767Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731}
156922024-02-27T21:50:10.331Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698317513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731}
156932024-02-27T21:50:10.340Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698354288Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731}
156942024-02-27T21:50:10.340Z{"msg":"Crucible b7531b92-bda8-4652-be2f-051727ba78a4 has session id: a1d19f26-8aa9-4331-93f4-42bd99ee12cb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698394493Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
156952024-02-27T21:50:10.340Z{"msg":"Upstairs opts: Upstairs UUID: b7531b92-bda8-4652-be2f-051727ba78a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698451527Z","hostname":"Crucible runtime is spawned
156962024-02-27T21:50:10.340Zip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
156972024-02-27T21:50:10.340Z{"msg":"Crucible stats registered with UUID: b7531b92-bda8-4652-be2f-051727ba78a4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.69850555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
156982024-02-27T21:50:10.340Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698598649Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731}
156992024-02-27T21:50:10.340Z{"msg":"b7531b92-bda8-4652-be2f-051727ba78a4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698713116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157002024-02-27T21:50:10.340Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698739613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157012024-02-27T21:50:10.340Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698776308Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157022024-02-27T21:50:10.340Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698792097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157032024-02-27T21:50:10.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698806275Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157042024-02-27T21:50:10.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698824333Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157052024-02-27T21:50:10.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698842221Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157062024-02-27T21:50:10.341Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698857959Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157072024-02-27T21:50:10.341Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698875357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157082024-02-27T21:50:10.341Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.698890295Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157092024-02-27T21:50:10.341Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.699060825Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157102024-02-27T21:50:10.341Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.699117948Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157112024-02-27T21:50:10.341Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.699247313Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157122024-02-27T21:50:10.341Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.699914384Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157132024-02-27T21:50:10.341Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.69995461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157142024-02-27T21:50:10.341Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.699971228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157152024-02-27T21:50:10.341Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.699985396Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157162024-02-27T21:50:10.341Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.700051568Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157172024-02-27T21:50:10.341Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.700073446Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157182024-02-27T21:50:10.341Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.700506545Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157192024-02-27T21:50:10.341Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.700591115Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157202024-02-27T21:50:10.341Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.700861923Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157212024-02-27T21:50:10.341Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701042961Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157222024-02-27T21:50:10.341Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701084076Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157232024-02-27T21:50:10.341Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701116423Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157242024-02-27T21:50:10.341Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701144369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157252024-02-27T21:50:10.341Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701169166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157262024-02-27T21:50:10.341Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701196273Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157272024-02-27T21:50:10.341Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701301201Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157282024-02-27T21:50:10.342Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701342696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157292024-02-27T21:50:10.342Z{"msg":"[0]R flush_numbers[0..12]: [255, 255, 265, 265, 265, 259, 259, 259, 261, 259, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701370623Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157302024-02-27T21:50:10.342Z{"msg":"[0]R generation[0..12]: [138, 138, 145, 145, 145, 142, 142, 142, 144, 142, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701401539Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157312024-02-27T21:50:10.342Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701427986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157322024-02-27T21:50:10.342Z{"msg":"[1]R flush_numbers[0..12]: [255, 255, 265, 265, 265, 259, 259, 259, 261, 259, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701452763Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157332024-02-27T21:50:10.342ZWait for a query_work_queue command to finish before sending IO
157342024-02-27T21:50:10.342Z{"msg":"[1]R generation[0..12]: [138, 138, 145, 145, 145, 142, 142, 142, 144, 142, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701485579Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157352024-02-27T21:50:10.342Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701512186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157362024-02-27T21:50:10.342Z{"msg":"[2]R flush_numbers[0..12]: [255, 255, 265, 265, 265, 259, 259, 259, 261, 259, 265, 265]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701545072Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157372024-02-27T21:50:10.342Z{"msg":"[2]R generation[0..12]: [138, 138, 145, 145, 145, 142, 142, 142, 144, 142, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701571279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157382024-02-27T21:50:10.342Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701610345Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157392024-02-27T21:50:10.342Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701633792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157402024-02-27T21:50:10.342Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701662728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157412024-02-27T21:50:10.342Z{"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701686526Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157422024-02-27T21:50:10.342Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701712833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"":"downstairs","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157432024-02-27T21:50:10.342Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.70173668Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157442024-02-27T21:50:10.342Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701762057Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"0","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157452024-02-27T21:50:10.342Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701785674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pidRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157462024-02-27T21:50:10.342Z":1731,"client":"1","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157472024-02-27T21:50:10.342Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701831668Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"client":"2","":"downstairs","session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157482024-02-27T21:50:10.342Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701865404Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157492024-02-27T21:50:10.342Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701888522Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157502024-02-27T21:50:10.342Z{"msg":"b7531b92-bda8-4652-be2f-051727ba78a4 is now active with session: a1d19f26-8aa9-4331-93f4-42bd99ee12cb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701915759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157512024-02-27T21:50:10.342Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701936996Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731,"session_id":"a1d19f26-8aa9-4331-93f4-42bd99ee12cb"}
157522024-02-27T21:50:10.342Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:08.701968662Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731}
157532024-02-27T21:50:10.343ZLoading write count information from file "/tmp/test_repair_verify.data"
157542024-02-27T21:50:10.343ZRun Repair workload
157552024-02-27T21:50:10.343Z01/30 Write block 400 len 16384 data: 13 11 9 8
157562024-02-27T21:50:10.348Z02/30 Write block 549 len 28672 data: 15 14 14 11 12 13 13
157572024-02-27T21:50:10.348Z03/30 Write block 14 len 36864 data: 14 15 15 14 13 12 12 12 9
157582024-02-27T21:50:10.349Z04/30 Write block 442 len 12288 data: 6 7 5
157592024-02-27T21:50:10.349Z05/30 Read block 538 len 8192
157602024-02-27T21:50:10.349Z06/30 Write block 304 len 4096 data: 14
157612024-02-27T21:50:10.349Z07/30 Write block 319 len 8192 data: 17 17
157622024-02-27T21:50:10.349Z08/30 Write block 236 len 20480 data: 18 11 11 11 7
157632024-02-27T21:50:10.349Z09/30 Write block 572 len 36864 data: 9 10 8 8 8 9 11 8 8
157642024-02-27T21:50:10.349Z10/30 Read block 19 len 12288
157652024-02-27T21:50:10.349Z11/30 Read block 529 len 36864
157662024-02-27T21:50:10.349Z12/30 Write block 116 len 40960 data: 8 10 12 13 15 17 15 16 16 12
157672024-02-27T21:50:10.349Z13/30 Flush
157682024-02-27T21:50:10.358Z14/30 Write block 403 len 20480 data: 9 9 8 8 6
157692024-02-27T21:50:10.358Z15/30 Read block 65 len 8192
157702024-02-27T21:50:10.358Z16/30 Read block 452 len 24576
157712024-02-27T21:50:10.358Z17/30 Read block 34 len 8192
157722024-02-27T21:50:10.358Z18/30 Write block 263 len 36864 data: 11 12 12 8 8 10 10 11 12
157732024-02-27T21:50:10.358Z19/30 Read block 101 len 8192
157742024-02-27T21:50:10.358Z20/30 Write block 87 len 36864 data: 12 13 14 11 11 8 11 10 11
157752024-02-27T21:50:10.364Z21/30 Write block 494 len 20480 data: 11 8 8 7 9
157762024-02-27T21:50:10.364Z22/30 Write block 148 len 12288 data: 9 9 9
157772024-02-27T21:50:10.364Z23/30 Flush
157782024-02-27T21:50:10.364Z24/30 Write block 188 len 4096 data: 18
157792024-02-27T21:50:10.364Z25/30 Read block 588 len 28672
157802024-02-27T21:50:10.364Z26/30 Read block 52 len 8192
157812024-02-27T21:50:10.364Z27/30 Write block 389 len 40960 data: 12 12 14 12 13 10 10 9 12 11
157822024-02-27T21:50:10.369Z28/30 Write block 16 len 20480 data: 16 15 14 13 13
157832024-02-27T21:50:10.369Z29/30 Write block 549 len 36864 data: 16 15 15 12 13 14 14 12 18
157842024-02-27T21:50:10.369Z30/30 Write block 123 len 8192 data: 17 17
157852024-02-27T21:50:10.369Z----------------------------------------------------------------
157862024-02-27T21:50:10.369Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
157872024-02-27T21:50:10.369ZStates: Active Active Active
157882024-02-27T21:50:10.369ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157892024-02-27T21:50:10.370Z 1 Acked 1000 Write 4 Done Done Done false
157902024-02-27T21:50:10.370Z 2 Acked 1001 Write 7 Done Done Sent false
157912024-02-27T21:50:10.370Z 3 Acked 1002 Write 9 Done Done Sent false
157922024-02-27T21:50:10.370Z 4 Acked 1003 Write 3 Done Done Sent false
157932024-02-27T21:50:10.370Z 5 Acked 1004 Read 2 Done Done Sent false
157942024-02-27T21:50:10.370Z 6 Acked 1005 Write 1 Done Done Sent false
157952024-02-27T21:50:10.370Z 7 Acked 1006 Write 2 Done Done Sent false
157962024-02-27T21:50:10.370Z 8 Acked 1007 Write 5 Done Done Sent false
157972024-02-27T21:50:10.370Z 9 Acked 1008 Write 9 Done Done Sent false
157982024-02-27T21:50:10.370Z 10 Acked 1009 Read 3 Done Done Sent false
157992024-02-27T21:50:10.370Z 11 Acked 1010 Read 9 Done Done Sent false
158002024-02-27T21:50:10.370Z 12 Acked 1011 Write 10 Done Done Sent false
158012024-02-27T21:50:10.370Z 13 Acked 1012 Flush 0 Done Done Sent false
158022024-02-27T21:50:10.370Z 14 Acked 1013 Write 5 Done Done Sent false
158032024-02-27T21:50:10.370Z 15 Acked 1014 Read 2 Done Done Sent false
158042024-02-27T21:50:10.370Z 16 Acked 1015 Read 6 Done Done Sent false
158052024-02-27T21:50:10.370Z 17 Acked 1016 Read 2 Done Done Sent false
158062024-02-27T21:50:10.370Z 18 Acked 1017 Write 9 Done Done Sent false
158072024-02-27T21:50:10.370Z 19 Acked 1018 Read 2 Done Done Sent false
158082024-02-27T21:50:10.370Z 20 Acked 1019 Write 9 Done Done Sent false
158092024-02-27T21:50:10.370Z 21 Acked 1020 Write 5 Done Done Sent false
158102024-02-27T21:50:10.370Z 22 Acked 1021 Write 3 Done Done Sent false
158112024-02-27T21:50:10.370Z 23 Acked 1022 Flush 0 Done Done Sent false
158122024-02-27T21:50:10.370Z 24 Acked 1023 Write 1 Done Done Sent false
158132024-02-27T21:50:10.370Z 25 Acked 1024 Read 7 Done Done Sent false
158142024-02-27T21:50:10.371Z 26 Acked 1025 Read 2 Done Done Sent false
158152024-02-27T21:50:10.371Z 27 Acked 1026 Write 10 Done Done Sent false
158162024-02-27T21:50:10.371Z 28 Acked 1027 Write 5 Done Done Sent false
158172024-02-27T21:50:10.371Z 29 Acked 1028 Write 9 Sent Sent Sent false
158182024-02-27T21:50:10.371Z 30 Acked 1029 Write 2 Sent Sent Sent false
158192024-02-27T21:50:10.371Z STATES DS:0 DS:1 DS:2 TOTAL
158202024-02-27T21:50:10.371Z New 0 0 0 0
158212024-02-27T21:50:10.371Z Sent 2 2 29 33
158222024-02-27T21:50:10.371Z Done 28 28 1 57
158232024-02-27T21:50:10.371Z Skipped 0 0 0 0
158242024-02-27T21:50:10.371Z Error 0 0 0 0
158252024-02-27T21:50:10.371ZLast Flush: 1022 1022 0
158262024-02-27T21:50:10.371ZDownstairs last five completed:
158272024-02-27T21:50:10.371ZUpstairs last five completed: 30 29 28 27 26
158282024-02-27T21:50:10.371Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:08.732515094Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1731}
158292024-02-27T21:50:10.371ZWrote out file "/tmp/test_repair_verify.data"
158302024-02-27T21:50:10.375Z
158312024-02-27T21:50:10.375ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158322024-02-27T21:50:10.803ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158332024-02-27T21:50:10.803Z 0 000-019 146 146 138 266 266 255 T T F <---
158342024-02-27T21:50:10.803Z 1 020-039 146 146 138 266 266 255 T T F <---
158352024-02-27T21:50:10.803Z 2 040-059 145 145 145 265 265 265 F F F
158362024-02-27T21:50:10.803Z 3 060-079 145 145 145 265 265 265 F F F
158372024-02-27T21:50:10.803Z 4 080-099 146 146 145 267 267 265 F F F <---
158382024-02-27T21:50:10.803Z 5 100-119 146 146 142 266 266 259 F F F <---
158392024-02-27T21:50:10.803Z 6 120-139 146 146 142 266 266 259 T T F <---
158402024-02-27T21:50:10.822Z 7 140-159 146 146 142 267 267 259 F F F <---
158412024-02-27T21:50:10.822Z 8 160-179 144 144 144 261 261 261 F F F
158422024-02-27T21:50:10.822Z 9 180-199 142 142 142 259 259 259 T T F <---
158432024-02-27T21:50:10.823Z 10 200-219 145 145 145 265 265 265 F F F
158442024-02-27T21:50:10.823Z 11 220-239 146 146 145 266 266 265 F F F <---
158452024-02-27T21:50:10.823Z 12 240-259 146 146 144 266 266 262 F F F <---
158462024-02-27T21:50:10.823Z 13 260-279 146 146 141 267 267 258 F F F <---
158472024-02-27T21:50:10.823Z 14 280-299 144 144 144 264 264 264 F F F
158482024-02-27T21:50:10.823Z 15 300-319 146 146 142 266 266 259 F F F <---
158492024-02-27T21:50:10.823Z 16 320-339 146 146 142 266 266 259 F F F <---
158502024-02-27T21:50:10.823Z 17 340-359 145 145 145 265 265 265 F F F
158512024-02-27T21:50:10.823Z 18 360-379 145 145 145 265 265 265 F F F
158522024-02-27T21:50:10.823Z 19 380-399 141 141 141 258 258 258 T T F <---
158532024-02-27T21:50:10.823Z 20 400-419 146 146 144 267 267 263 F F T <---
158542024-02-27T21:50:10.823Z 21 420-439 144 144 144 263 263 263 F F F
158552024-02-27T21:50:10.823Z 22 440-459 146 146 144 266 266 261 F F F <---
158562024-02-27T21:50:10.823Z 23 460-479 141 141 141 258 258 258 F F F
158572024-02-27T21:50:10.824Z 24 480-499 146 146 141 267 267 258 F F F <---
158582024-02-27T21:50:10.824Z 25 500-519 144 144 144 262 262 262 F F F
158592024-02-27T21:50:10.824Z 26 520-539 142 142 142 259 259 259 F F F
158602024-02-27T21:50:10.824Z 27 540-559 146 146 144 266 266 264 T T F <---
158612024-02-27T21:50:10.824Z 28 560-579 146 146 142 266 266 259 F F F <---
158622024-02-27T21:50:10.824Z 29 580-599 146 146 138 266 266 254 F F F <---
158632024-02-27T21:50:10.824ZMax gen: 146, Max flush: 267
158642024-02-27T21:50:10.824ZError: Difference in extent metadata found!
158652024-02-27T21:50:10.824ZA Difference in extent metadata is expected here
158662024-02-27T21:50:10.824ZOn loop 73
158672024-02-27T21:50:10.824Z
158682024-02-27T21:50:10.824ZVerifying data now
158692024-02-27T21:50:15.296ZNo change after verify
158702024-02-27T21:50:15.296ZLoop: 73 Downstairs dump after verify (and repair):
158712024-02-27T21:50:15.310ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158722024-02-27T21:50:15.310Z 0 000-019 147 147 147 268 268 268 F F F
158732024-02-27T21:50:15.310Z 1 020-039 147 147 147 268 268 268 F F F
158742024-02-27T21:50:15.310Z 2 040-059 145 145 145 265 265 265 F F F
158752024-02-27T21:50:15.310Z 3 060-079 145 145 145 265 265 265 F F F
158762024-02-27T21:50:15.310Z 4 080-099 146 146 146 267 267 267 F F F
158772024-02-27T21:50:15.314Z 5 100-119 146 146 146 266 266 266 F F F
158782024-02-27T21:50:15.315Z 6 120-139 147 147 147 268 268 268 F F F
158792024-02-27T21:50:15.315Z 7 140-159 146 146 146 267 267 267 F F F
158802024-02-27T21:50:15.315Z 8 160-179 144 144 144 261 261 261 F F F
158812024-02-27T21:50:15.315Z 9 180-199 147 147 147 268 268 268 F F F
158822024-02-27T21:50:15.315Z 10 200-219 145 145 145 265 265 265 F F F
158832024-02-27T21:50:15.315Z 11 220-239 146 146 146 266 266 266 F F F
158842024-02-27T21:50:15.315Z 12 240-259 146 146 146 266 266 266 F F F
158852024-02-27T21:50:15.315Z 13 260-279 146 146 146 267 267 267 F F F
158862024-02-27T21:50:15.315Z 14 280-299 144 144 144 264 264 264 F F F
158872024-02-27T21:50:15.315Z 15 300-319 146 146 146 266 266 266 F F F
158882024-02-27T21:50:15.315Z 16 320-339 146 146 146 266 266 266 F F F
158892024-02-27T21:50:15.316Z 17 340-359 145 145 145 265 265 265 F F F
158902024-02-27T21:50:15.316Z 18 360-379 145 145 145 265 265 265 F F F
158912024-02-27T21:50:15.316Z 19 380-399 147 147 147 268 268 268 F F F
158922024-02-27T21:50:15.316Z 20 400-419 146 146 146 267 267 267 F F F
158932024-02-27T21:50:15.316Z 21 420-439 144 144 144 263 263 263 F F F
158942024-02-27T21:50:15.316Z 22 440-459 146 146 146 266 266 266 F F F
158952024-02-27T21:50:15.316Z 23 460-479 141 141 141 258 258 258 F F F
158962024-02-27T21:50:15.316Z 24 480-499 146 146 146 267 267 267 F F F
158972024-02-27T21:50:15.316Z 25 500-519 144 144 144 262 262 262 F F F
158982024-02-27T21:50:15.316Z 26 520-539 142 142 142 259 259 259 F F F
158992024-02-27T21:50:15.316Z 27 540-559 147 147 147 268 268 268 F F F
159002024-02-27T21:50:15.316Z 28 560-579 146 146 146 266 266 266 F F F
159012024-02-27T21:50:15.317Z 29 580-599 146 146 146 266 266 266 F F F
159022024-02-27T21:50:15.317ZMax gen: 147, Max flush: 268
159032024-02-27T21:50:15.317Z
159042024-02-27T21:50:15.317ZBegin loop 74
159052024-02-27T21:50:15.317ZDownstairs to restart: 2
159062024-02-27T21:50:15.331Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.698330459Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739}
159072024-02-27T21:50:15.332Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,Crucible runtime is spawned
159082024-02-27T21:50:15.332Z"time":"2024-02-27T21:50:13.69865964Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739}
159092024-02-27T21:50:15.339Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.698723352Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739}
159102024-02-27T21:50:15.339Z{"msg":"Crucible 4e3520b9-a5dc-424c-ba6b-bbbeef5287fc has session id: 1e55202f-60c9-4a94-9d4e-2798068295f9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.698755429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159112024-02-27T21:50:15.339Z{"msg":"Upstairs opts: Upstairs UUID: 4e3520b9-a5dc-424c-ba6b-bbbeef5287fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.698797754Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159122024-02-27T21:50:15.339Z{"msg":"Crucible stats registered with UUID: 4e3520b9-a5dc-424c-ba6b-bbbeef5287fc","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.698841658Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159132024-02-27T21:50:15.339Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.698870075Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739}
159142024-02-27T21:50:15.339Z{"msg":"4e3520b9-a5dc-424c-ba6b-bbbeef5287fc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.698928688Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159152024-02-27T21:50:15.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.698975413Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159162024-02-27T21:50:15.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699034836Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159172024-02-27T21:50:15.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.6990856Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159182024-02-27T21:50:15.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699120446Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159192024-02-27T21:50:15.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699162941Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159202024-02-27T21:50:15.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699211405Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159212024-02-27T21:50:15.339Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699245961Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159222024-02-27T21:50:15.339Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699298665Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159232024-02-27T21:50:15.339Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699333201Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159242024-02-27T21:50:15.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699361637Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159252024-02-27T21:50:15.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699389124Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159262024-02-27T21:50:15.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.699411731Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159272024-02-27T21:50:15.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.700024809Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159282024-02-27T21:50:15.339Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.700062165Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159292024-02-27T21:50:15.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.700078473Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159302024-02-27T21:50:15.340Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.700094161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159312024-02-27T21:50:15.340Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.700119448Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159322024-02-27T21:50:15.340Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.700146125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159332024-02-27T21:50:15.340Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.700606101Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159342024-02-27T21:50:15.340Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.700720847Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159352024-02-27T21:50:15.340Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.700770391Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159362024-02-27T21:50:15.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701107512Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159372024-02-27T21:50:15.340Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701158716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159382024-02-27T21:50:15.340Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701231817Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159392024-02-27T21:50:15.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.70129026Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159402024-02-27T21:50:15.340Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701318967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159412024-02-27T21:50:15.340Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701334065Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159422024-02-27T21:50:15.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701353493Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159432024-02-27T21:50:15.340Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701367441Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159442024-02-27T21:50:15.340Z{"msg":"[0]R flush_numbers[0..12]: [268, 268, 265, 265, 267, 266, 268, 267, 261, 268, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701388489Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159452024-02-27T21:50:15.340Z{"msg":"[0]R generation[0..12]: [147, 147, 145, 145, 146, 146, 147, 146, 144, 147, 145, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701403027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159462024-02-27T21:50:15.340Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701424044Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159472024-02-27T21:50:15.340Z{"msg":"[1]R flush_numbers[0..12]: [268, 268, 265, 265, 267, 266, 268, 267, 261, 268, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701438973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159482024-02-27T21:50:15.340Z{"msg":"[1]R generation[0..12]: [147, 147, 145, 145, 146, 146, 147, 146, 144, 147, 145, 146]","v":0,"Wait for a query_work_queue command to finish before sending IO
159492024-02-27T21:50:15.340Zname":"crucible","level":30,"time":"2024-02-27T21:50:13.70146227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159502024-02-27T21:50:15.340Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701477108Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159512024-02-27T21:50:15.340Z{"msg":"[2]R flush_numbers[0..12]: [268, 268, 265, 265, 267, 266, 268, 267, 261, 268, 265, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701502645Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159522024-02-27T21:50:15.340Z{"msg":"[2]R generation[0..12]: [147, 147, 145, 145, 146, 146, 147, 146, 144, 147, 145, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701536051Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159532024-02-27T21:50:15.340Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701566788Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159542024-02-27T21:50:15.341Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701585995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159552024-02-27T21:50:15.341Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701620111Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159562024-02-27T21:50:15.341Z{"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701655747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159572024-02-27T21:50:15.341Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701680094Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"":"downstairs","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159582024-02-27T21:50:15.341Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159592024-02-27T21:50:15.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701751686Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"0","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159602024-02-27T21:50:15.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701779292Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"1","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159612024-02-27T21:50:15.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701812069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"client":"2","":"downstairs","session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159622024-02-27T21:50:15.341Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701828167Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159632024-02-27T21:50:15.341Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701847544Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159642024-02-27T21:50:15.341Z{"msg":"4e3520b9-a5dc-424c-ba6b-bbbeef5287fc is now active with session: 1e55202f-60c9-4a94-9d4e-2798068295f9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701868942Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159652024-02-27T21:50:15.341ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159662024-02-27T21:50:15.341Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701905228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739,"session_id":"1e55202f-60c9-4a94-9d4e-2798068295f9"}
159672024-02-27T21:50:15.342Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:13.701919396Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739}
159682024-02-27T21:50:15.342ZLoading write count information from file "/tmp/test_repair_verify.data"
159692024-02-27T21:50:15.342ZRun Repair workload
159702024-02-27T21:50:15.348Z01/30 Write block 24 len 36864 data: 11 10 10 10 10 10 12 11 10
159712024-02-27T21:50:15.348Z02/30 Read block 457 len 8192
159722024-02-27T21:50:15.349Z03/30 Write block 454 len 4096 data: 8
159732024-02-27T21:50:15.349Z04/30 Write block 511 len 8192 data: 18 17
159742024-02-27T21:50:15.349Z05/30 Write block 495 len 8192 data: 9 9
159752024-02-27T21:50:15.349Z06/30 Read block 148 len 24576
159762024-02-27T21:50:15.349Z07/30 Flush
159772024-02-27T21:50:15.349Z08/30 Write block 577 len 20480 data: 10 12 9 9 8
159782024-02-27T21:50:15.349Z09/30 Write block 197 len 36864 data: 8 7 9 10 9 8 8 8 5
159792024-02-27T21:50:15.349Z10/30 Read block 97 len 20480
159802024-02-27T21:50:15.349Z11/30 Read block 347 len 40960
159812024-02-27T21:50:15.353Z12/30 Flush
159822024-02-27T21:50:15.353Z13/30 Write block 113 len 36864 data: 11 9 9 9 11 13 14 16 18
159832024-02-27T21:50:15.353Z14/30 Read block 206 len 24576
159842024-02-27T21:50:15.353Z15/30 Write block 590 len 16384 data: 13 10 12 12
159852024-02-27T21:50:15.367Z16/30 Write block 254 len 16384 data: 12 16 15 11
159862024-02-27T21:50:15.367Z17/30 Read block 241 len 4096
159872024-02-27T21:50:15.367Z18/30 Read block 449 len 40960
159882024-02-27T21:50:15.367Z19/30 Write block 251 len 20480 data: 13 13 13 13 17
159892024-02-27T21:50:15.367Z20/30 Read block 440 len 12288
159902024-02-27T21:50:15.372Z21/30 Read block 97 len 20480
159912024-02-27T21:50:15.372Z22/30 Read block 597 len 12288
159922024-02-27T21:50:15.372Z23/30 Write block 526 len 16384 data: 9 9 9 7
159932024-02-27T21:50:15.372Z24/30 Read block 204 len 8192
159942024-02-27T21:50:15.372Z25/30 Read block 501 len 20480
159952024-02-27T21:50:15.372Z26/30 Read block 202 len 28672
159962024-02-27T21:50:15.372Z27/30 Write block 130 len 32768 data: 10 8 5 6 6 8 9 10
159972024-02-27T21:50:15.372Z28/30 Write block 168 len 24576 data: 5 7 8 8 8 7
159982024-02-27T21:50:15.372Z29/30 Read block 28 len 28672
159992024-02-27T21:50:15.372Z30/30 Read block 533 len 20480
160002024-02-27T21:50:15.372Z----------------------------------------------------------------
160012024-02-27T21:50:15.372Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
160022024-02-27T21:50:15.373ZStates: Active Active Active
160032024-02-27T21:50:15.373ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
160042024-02-27T21:50:15.373Z 1 Acked 1000 Write 9 Done Done Done false
160052024-02-27T21:50:15.373Z 2 Acked 1001 Read 2 Done Done Done false
160062024-02-27T21:50:15.373Z 3 Acked 1002 Write 1 Done Done Sent false
160072024-02-27T21:50:15.373Z 4 Acked 1003 Write 2 Done Done Sent false
160082024-02-27T21:50:15.373Z 5 Acked 1004 Write 2 Done Done Sent false
160092024-02-27T21:50:15.373Z 6 Acked 1005 Read 6 Done Done Sent false
160102024-02-27T21:50:15.373Z 7 Acked 1006 Flush 0 Done Done Sent false
160112024-02-27T21:50:15.373Z 8 Acked 1007 Write 5 Done Done Sent false
160122024-02-27T21:50:15.373Z 9 Acked 1008 Write 9 Done Done Sent false
160132024-02-27T21:50:15.373Z 10 Acked 1009 Read 5 Done Done Sent false
160142024-02-27T21:50:15.373Z 11 Acked 1010 Read 10 Done Done Sent false
160152024-02-27T21:50:15.374Z 12 Acked 1011 Flush 0 Done Done Sent false
160162024-02-27T21:50:15.374Z 13 Acked 1012 Write 9 Done Done Sent false
160172024-02-27T21:50:15.374Z 14 Acked 1013 Read 6 Done Done Sent false
160182024-02-27T21:50:15.374Z 15 Acked 1014 Write 4 Done Done Sent false
160192024-02-27T21:50:15.374Z 16 Acked 1015 Write 4 Done Done Sent false
160202024-02-27T21:50:15.374Z 17 Acked 1016 Read 1 Done Done Sent false
160212024-02-27T21:50:15.374Z 18 Acked 1017 Read 10 Done Done Sent false
160222024-02-27T21:50:15.374Z 19 Acked 1018 Write 5 Done Done Sent false
160232024-02-27T21:50:15.374Z 20 Acked 1019 Read 3 Done Done Sent false
160242024-02-27T21:50:15.374Z 21 Acked 1020 Read 5 Done Done Sent false
160252024-02-27T21:50:15.374Z 22 Acked 1021 Read 3 Done Done Sent false
160262024-02-27T21:50:15.374Z 23 Acked 1022 Write 4 Done Done Sent false
160272024-02-27T21:50:15.374Z 24 Acked 1023 Read 2 Done Done Sent false
160282024-02-27T21:50:15.375Z 25 Acked 1024 Read 5 Done Done Sent false
160292024-02-27T21:50:15.375Z 26 Acked 1025 Read 7 Done Done Sent false
160302024-02-27T21:50:15.375Z 27 Acked 1026 Write 8 Done Done Sent false
160312024-02-27T21:50:15.375Z 28 Acked 1027 Write 6 Done Done Sent false
160322024-02-27T21:50:15.375Z 29 Acked 1028 Read 7 Done Done Sent false
160332024-02-27T21:50:15.375Z 30 Acked 1029 Read 5 Done Sent Sent false
160342024-02-27T21:50:15.375Z STATES DS:0 DS:1 DS:2 TOTAL
160352024-02-27T21:50:15.375Z New 0 0 0 0
160362024-02-27T21:50:15.375Z Sent 0 1 28 29
160372024-02-27T21:50:15.375Z Done 30 29 2 61
160382024-02-27T21:50:15.375Z Skipped 0 0 0 0
160392024-02-27T21:50:15.375Z Error 0 0 0 0
160402024-02-27T21:50:15.375ZLast Flush: 1011 1011 0
160412024-02-27T21:50:15.376ZDownstairs last five completed:
160422024-02-27T21:50:15.376ZUpstairs last five completed: 30 29 28 27 26
160432024-02-27T21:50:15.376Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:13.731919302Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1739}
160442024-02-27T21:50:15.376ZWrote out file "/tmp/test_repair_verify.data"
160452024-02-27T21:50:15.376Z
160462024-02-27T21:50:15.380ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160472024-02-27T21:50:15.389ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160482024-02-27T21:50:15.390Z 0 000-019 147 147 147 268 268 268 F F F
160492024-02-27T21:50:15.390Z 1 020-039 148 148 147 269 269 268 F F T <---
160502024-02-27T21:50:15.390Z 2 040-059 145 145 145 265 265 265 F F F
160512024-02-27T21:50:15.390Z 3 060-079 145 145 145 265 265 265 F F F
160522024-02-27T21:50:15.390Z 4 080-099 146 146 146 267 267 267 F F F
160532024-02-27T21:50:15.390Z 5 100-119 146 146 146 266 266 266 T T F <---
160542024-02-27T21:50:15.393Z 6 120-139 147 147 147 268 268 268 T T F <---
160552024-02-27T21:50:15.393Z 7 140-159 146 146 146 267 267 267 F F F
160562024-02-27T21:50:15.394Z 8 160-179 144 144 144 261 261 261 T T F <---
160572024-02-27T21:50:15.394Z 9 180-199 148 148 147 270 270 268 F F F <---
160582024-02-27T21:50:15.394Z 10 200-219 148 148 145 270 270 265 F F F <---
160592024-02-27T21:50:15.394Z 11 220-239 146 146 146 266 266 266 F F F
160602024-02-27T21:50:15.394Z 12 240-259 146 146 146 266 266 266 T T F <---
160612024-02-27T21:50:15.394Z 13 260-279 146 146 146 267 267 267 F F F
160622024-02-27T21:50:15.394Z 14 280-299 144 144 144 264 264 264 F F F
160632024-02-27T21:50:15.394Z 15 300-319 146 146 146 266 266 266 F F F
160642024-02-27T21:50:15.394Z 16 320-339 146 146 146 266 266 266 F F F
160652024-02-27T21:50:15.394Z 17 340-359 145 145 145 265 265 265 F F F
160662024-02-27T21:50:15.394Z 18 360-379 145 145 145 265 265 265 F F F
160672024-02-27T21:50:15.394Z 19 380-399 147 147 147 268 268 268 F F F
160682024-02-27T21:50:15.394Z 20 400-419 146 146 146 267 267 267 F F F
160692024-02-27T21:50:15.394Z 21 420-439 144 144 144 263 263 263 F F F
160702024-02-27T21:50:15.394Z 22 440-459 148 148 146 269 269 266 F F F <---
160712024-02-27T21:50:15.394Z 23 460-479 141 141 141 258 258 258 F F F
160722024-02-27T21:50:15.394Z 24 480-499 148 148 146 269 269 267 F F F <---
160732024-02-27T21:50:15.394Z 25 500-519 148 148 144 269 269 262 F F F <---
160742024-02-27T21:50:15.394Z 26 520-539 142 142 142 259 259 259 T T F <---
160752024-02-27T21:50:15.394Z 27 540-559 147 147 147 268 268 268 F F F
160762024-02-27T21:50:15.394Z 28 560-579 148 148 146 270 270 266 F F F <---
160772024-02-27T21:50:15.394Z 29 580-599 148 148 146 270 270 266 T T F <---
160782024-02-27T21:50:15.394ZMax gen: 148, Max flush: 270
160792024-02-27T21:50:15.395ZError: Difference in extent metadata found!
160802024-02-27T21:50:15.395ZA Difference in extent metadata is expected here
160812024-02-27T21:50:15.395ZOn loop 74
160822024-02-27T21:50:15.395Z
160832024-02-27T21:50:15.398ZVerifying data now
160842024-02-27T21:50:20.049ZNo change after verify
160852024-02-27T21:50:20.050ZLoop: 74 Downstairs dump after verify (and repair):
160862024-02-27T21:50:20.063ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160872024-02-27T21:50:20.063Z 0 000-019 147 147 147 268 268 268 F F F
160882024-02-27T21:50:20.063Z 1 020-039 148 148 148 269 269 269 F F F
160892024-02-27T21:50:20.063Z 2 040-059 145 145 145 265 265 265 F F F
160902024-02-27T21:50:20.063Z 3 060-079 145 145 145 265 265 265 F F F
160912024-02-27T21:50:20.063Z 4 080-099 146 146 146 267 267 267 F F F
160922024-02-27T21:50:20.067Z 5 100-119 149 149 149 271 271 271 F F F
160932024-02-27T21:50:20.067Z 6 120-139 149 149 149 271 271 271 F F F
160942024-02-27T21:50:20.067Z 7 140-159 146 146 146 267 267 267 F F F
160952024-02-27T21:50:20.067Z 8 160-179 149 149 149 271 271 271 F F F
160962024-02-27T21:50:20.067Z 9 180-199 148 148 148 270 270 270 F F F
160972024-02-27T21:50:20.067Z 10 200-219 148 148 148 270 270 270 F F F
160982024-02-27T21:50:20.067Z 11 220-239 146 146 146 266 266 266 F F F
160992024-02-27T21:50:20.068Z 12 240-259 149 149 149 271 271 271 F F F
161002024-02-27T21:50:20.068Z 13 260-279 146 146 146 267 267 267 F F F
161012024-02-27T21:50:20.068Z 14 280-299 144 144 144 264 264 264 F F F
161022024-02-27T21:50:20.068Z 15 300-319 146 146 146 266 266 266 F F F
161032024-02-27T21:50:20.068Z 16 320-339 146 146 146 266 266 266 F F F
161042024-02-27T21:50:20.068Z 17 340-359 145 145 145 265 265 265 F F F
161052024-02-27T21:50:20.068Z 18 360-379 145 145 145 265 265 265 F F F
161062024-02-27T21:50:20.068Z 19 380-399 147 147 147 268 268 268 F F F
161072024-02-27T21:50:20.068Z 20 400-419 146 146 146 267 267 267 F F F
161082024-02-27T21:50:20.068Z 21 420-439 144 144 144 263 263 263 F F F
161092024-02-27T21:50:20.068Z 22 440-459 148 148 148 269 269 269 F F F
161102024-02-27T21:50:20.068Z 23 460-479 141 141 141 258 258 258 F F F
161112024-02-27T21:50:20.068Z 24 480-499 148 148 148 269 269 269 F F F
161122024-02-27T21:50:20.068Z 25 500-519 148 148 148 269 269 269 F F F
161132024-02-27T21:50:20.068Z 26 520-539 149 149 149 271 271 271 F F F
161142024-02-27T21:50:20.068Z 27 540-559 147 147 147 268 268 268 F F F
161152024-02-27T21:50:20.068Z 28 560-579 148 148 148 270 270 270 F F F
161162024-02-27T21:50:20.068Z 29 580-599 149 149 149 271 271 271 F F F
161172024-02-27T21:50:20.068ZMax gen: 149, Max flush: 271
161182024-02-27T21:50:20.068Z
161192024-02-27T21:50:20.068ZBegin loop 75
161202024-02-27T21:50:20.068ZDownstairs to restart: 0
161212024-02-27T21:50:20.086Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452338974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747}
161222024-02-27T21:50:20.086Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
161232024-02-27T21:50:20.086Z,"time":"2024-02-27T21:50:18.452652577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747}
161242024-02-27T21:50:20.093Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452698581Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747}
161252024-02-27T21:50:20.093Z{"msg":"Crucible 3522e295-9f59-41db-8dfc-60e70b108033 has session id: efe97bef-34a7-4054-9ae6-293a3b4882b9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452731647Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161262024-02-27T21:50:20.093Z{"msg":"Upstairs opts: Upstairs UUID: 3522e295-9f59-41db-8dfc-60e70b108033, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452785451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161272024-02-27T21:50:20.093Z{"msg":"Crucible stats registered with UUID: 3522e295-9f59-41db-8dfc-60e70b108033","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452815228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161282024-02-27T21:50:20.093Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452841385Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747}
161292024-02-27T21:50:20.093Z{"msg":"3522e295-9f59-41db-8dfc-60e70b108033 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452853293Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161302024-02-27T21:50:20.094Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452865702Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161312024-02-27T21:50:20.094Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452900987Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161322024-02-27T21:50:20.094Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452927095Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161332024-02-27T21:50:20.094Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.45296279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161342024-02-27T21:50:20.094Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.452985537Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161352024-02-27T21:50:20.094Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.453009735Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161362024-02-27T21:50:20.094Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.453034982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161372024-02-27T21:50:20.094Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.453247767Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161382024-02-27T21:50:20.094Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.453329777Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161392024-02-27T21:50:20.094Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.453403958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161402024-02-27T21:50:20.094Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.453447973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161412024-02-27T21:50:20.094Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.453490758Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161422024-02-27T21:50:20.094Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.453942165Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161432024-02-27T21:50:20.094Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.453997208Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161442024-02-27T21:50:20.094Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.454031754Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161452024-02-27T21:50:20.094Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.454059401Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161462024-02-27T21:50:20.094Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.454167488Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161472024-02-27T21:50:20.094Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.454202624Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161482024-02-27T21:50:20.094Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.454651711Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161492024-02-27T21:50:20.094Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.454689307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161502024-02-27T21:50:20.094Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.454706365Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161512024-02-27T21:50:20.094Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455168011Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161522024-02-27T21:50:20.094Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455217974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161532024-02-27T21:50:20.094Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455240662Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161542024-02-27T21:50:20.095Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455268759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161552024-02-27T21:50:20.095Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455297885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161562024-02-27T21:50:20.095Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455319363Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161572024-02-27T21:50:20.095Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455353359Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161582024-02-27T21:50:20.095Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455369717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161592024-02-27T21:50:20.095Z{"msg":"[0]R flush_numbers[0..12]: [268, 269, 265, 265, 267, 271, 271, 267, 271, 270, 270, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455397664Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161602024-02-27T21:50:20.095Z{"msg":"[0]R generation[0..12]: [147, 148, 145, 145, 146, 149, 149, 146, 149, 148, 148, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455413771Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161612024-02-27T21:50:20.095Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455436339Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161622024-02-27T21:50:20.095Z{"msg":"[1]R flush_numbers[0..12]: [268, 269, 265, 265, 267, 271, 271, 267, 271, 270, 270, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455451027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161632024-02-27T21:50:20.095Z{"msg":"[1]R generation[0..12]: [147, 148, 145, 145, 146, 149, 149, 146, 149, 148, 148, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455472335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161642024-02-27T21:50:20.095Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455487043Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161652024-02-27T21:50:20.095Z{"msg":"[2]R flush_numbers[0..12]: [268, 269, 265, 265, 267, 271, 271, 267, 271, 270, 270, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.45550839Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161662024-02-27T21:50:20.095Z{"msg":"[2]R generation[0..12]: [147, 148, 145, 145, 146, 149, 149, 146, 149, 148, 148, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455522909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161672024-02-27T21:50:20.095ZWait for a query_work_queue command to finish before sending IO
161682024-02-27T21:50:20.095Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455548736Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161692024-02-27T21:50:20.095Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455570613Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161702024-02-27T21:50:20.095Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455602369Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161712024-02-27T21:50:20.095Z{"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455617838Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161722024-02-27T21:50:20.095Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455645214Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"":"downstairs","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161732024-02-27T21:50:20.095Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.45567764Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161742024-02-27T21:50:20.095Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455702497Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"0","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161752024-02-27T21:50:20.095Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455734354Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"1","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161762024-02-27T21:50:20.095Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.45576797Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"client":"2","":"downstairs","session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161772024-02-27T21:50:20.095Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161782024-02-27T21:50:20.095Z,"time":"2024-02-27T21:50:18.455800376Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161792024-02-27T21:50:20.096Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455824333Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161802024-02-27T21:50:20.096Z{"msg":"3522e295-9f59-41db-8dfc-60e70b108033 is now active with session: efe97bef-34a7-4054-9ae6-293a3b4882b9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455856379Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161812024-02-27T21:50:20.096Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455871577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747,"session_id":"efe97bef-34a7-4054-9ae6-293a3b4882b9"}
161822024-02-27T21:50:20.096Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:18.455891945Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747}
161832024-02-27T21:50:20.099ZLoading write count information from file "/tmp/test_repair_verify.data"
161842024-02-27T21:50:20.099ZRun Repair workload
161852024-02-27T21:50:20.099Z01/30 Write block 98 len 40960 data: 8 12 13 11 13 12 10 12 15 17
161862024-02-27T21:50:20.100Z02/30 Read block 375 len 40960
161872024-02-27T21:50:20.100Z03/30 Read block 574 len 12288
161882024-02-27T21:50:20.100Z04/30 Write block 195 len 24576 data: 10 8 9 8 10 11
161892024-02-27T21:50:20.100Z05/30 Read block 411 len 4096
161902024-02-27T21:50:20.100Z06/30 Read block 593 len 4096
161912024-02-27T21:50:20.100Z07/30 Read block 525 len 20480
161922024-02-27T21:50:20.100Z08/30 Read block 295 len 4096
161932024-02-27T21:50:20.100Z09/30 Write block 327 len 36864 data: 9 10 11 11 12 12 14 14 17
161942024-02-27T21:50:20.100Z10/30 Read block 85 len 8192
161952024-02-27T21:50:20.103Z11/30 Write block 287 len 36864 data: 9 10 13 13 13 12 15 15 16
161962024-02-27T21:50:20.103Z12/30 Write block 336 len 24576 data: 13 13 15 14 17 17
161972024-02-27T21:50:20.103Z13/30 Write block 293 len 40960 data: 16 16 17 15 15 14 10 12 13 13
161982024-02-27T21:50:20.103Z14/30 Read block 263 len 40960
161992024-02-27T21:50:20.107Z15/30 Read block 446 len 20480
162002024-02-27T21:50:20.107Z16/30 Write block 64 len 20480 data: 8 8 8 9 9
162012024-02-27T21:50:20.107Z17/30 Write block 302 len 16384 data: 14 11 15 12
162022024-02-27T21:50:20.107Z18/30 Read block 511 len 32768
162032024-02-27T21:50:20.110Z19/30 Write block 48 len 24576 data: 12 12 13 13 14 13
162042024-02-27T21:50:20.110Z20/30 Read block 9 len 32768
162052024-02-27T21:50:20.110Z21/30 Write block 149 len 16384 data: 10 10 8 8
162062024-02-27T21:50:20.113Z22/30 Write block 210 len 28672 data: 11 11 12 10 12 14 15
162072024-02-27T21:50:20.113Z23/30 Read block 284 len 36864
162082024-02-27T21:50:20.113Z24/30 Read block 51 len 28672
162092024-02-27T21:50:20.113Z25/30 Write block 395 len 12288 data: 11 10 13
162102024-02-27T21:50:20.116Z26/30 Write block 92 len 4096 data: 9
162112024-02-27T21:50:20.116Z27/30 Write block 264 len 12288 data: 13 13 9
162122024-02-27T21:50:20.116Z28/30 Write block 551 len 16384 data: 16 13 14 15
162132024-02-27T21:50:20.116Z29/30 Read block 237 len 24576
162142024-02-27T21:50:20.116Z30/30 Read block 445 len 20480
162152024-02-27T21:50:20.123Z----------------------------------------------------------------
162162024-02-27T21:50:20.123Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
162172024-02-27T21:50:20.123ZStates: Active Active Active
162182024-02-27T21:50:20.123ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162192024-02-27T21:50:20.123Z 1 Acked 1000 Write 10 Done Done Done false
162202024-02-27T21:50:20.124Z 2 Acked 1001 Read 10 Done Done Done false
162212024-02-27T21:50:20.124Z 3 Acked 1002 Read 3 Sent Done Done false
162222024-02-27T21:50:20.124Z 4 Acked 1003 Write 6 Sent Done Done false
162232024-02-27T21:50:20.124Z 5 Acked 1004 Read 1 Sent Done Done false
162242024-02-27T21:50:20.124Z 6 Acked 1005 Read 1 Sent Done Done false
162252024-02-27T21:50:20.124Z 7 Acked 1006 Read 5 Sent Done Done false
162262024-02-27T21:50:20.124Z 8 Acked 1007 Read 1 Sent Done Done false
162272024-02-27T21:50:20.124Z 9 Acked 1008 Write 9 Sent Done Done false
162282024-02-27T21:50:20.124Z 10 Acked 1009 Read 2 Sent Done Done false
162292024-02-27T21:50:20.124Z 11 Acked 1010 Write 9 Sent Done Done false
162302024-02-27T21:50:20.124Z 12 Acked 1011 Write 6 Sent Done Done false
162312024-02-27T21:50:20.124Z 13 Acked 1012 Write 10 Sent Done Done false
162322024-02-27T21:50:20.124Z 14 Acked 1013 Read 10 Sent Done Done false
162332024-02-27T21:50:20.124Z 15 Acked 1014 Read 5 Sent Done Done false
162342024-02-27T21:50:20.124Z 16 Acked 1015 Write 5 Sent Done Done false
162352024-02-27T21:50:20.124Z 17 Acked 1016 Write 4 Sent Done Done false
162362024-02-27T21:50:20.124Z 18 Acked 1017 Read 8 Sent Done Done false
162372024-02-27T21:50:20.124Z 19 Acked 1018 Write 6 Sent Done Done false
162382024-02-27T21:50:20.124Z 20 Acked 1019 Read 8 Sent Done Done false
162392024-02-27T21:50:20.124Z 21 Acked 1020 Write 4 Sent Done Done false
162402024-02-27T21:50:20.124Z 22 Acked 1021 Write 7 Sent Done Done false
162412024-02-27T21:50:20.124Z 23 Acked 1022 Read 9 Sent Done Done false
162422024-02-27T21:50:20.124Z 24 Acked 1023 Read 7 Sent Done Done false
162432024-02-27T21:50:20.124Z 25 Acked 1024 Write 3 Sent Done Done false
162442024-02-27T21:50:20.125Z 26 Acked 1025 Write 1 Sent Done Done false
162452024-02-27T21:50:20.125Z 27 Acked 1026 Write 3 Sent Done Done false
162462024-02-27T21:50:20.125Z 28 Acked 1027 Write 4 Sent Done Done false
162472024-02-27T21:50:20.125Z 29 Acked 1028 Read 6 Sent Done Done false
162482024-02-27T21:50:20.125Z 30 Acked 1029 Read 5 Sent Done Done false
162492024-02-27T21:50:20.125Z STATES DS:0 DS:1 DS:2 TOTAL
162502024-02-27T21:50:20.125Z New 0 0 0 0
162512024-02-27T21:50:20.125Z Sent 28 0 0 28
162522024-02-27T21:50:20.125Z Done 2 30 30 62
162532024-02-27T21:50:20.125Z Skipped 0 0 0 0
162542024-02-27T21:50:20.125Z Error 0 0 0 0
162552024-02-27T21:50:20.125ZLast Flush: 0 0 0
162562024-02-27T21:50:20.125ZDownstairs last five completed:
162572024-02-27T21:50:20.125ZUpstairs last five completed: 30 29 28 27 26
162582024-02-27T21:50:20.125Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:18.484171654Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1747}
162592024-02-27T21:50:20.125ZWrote out file "/tmp/test_repair_verify.data"
162602024-02-27T21:50:20.129Z
162612024-02-27T21:50:20.129ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162622024-02-27T21:50:20.145ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162632024-02-27T21:50:20.145Z 0 000-019 147 147 147 268 268 268 F F F
162642024-02-27T21:50:20.145Z 1 020-039 148 148 148 269 269 269 F F F
162652024-02-27T21:50:20.145Z 2 040-059 145 145 145 265 265 265 F T T <---
162662024-02-27T21:50:20.145Z 3 060-079 145 145 145 265 265 265 F T T <---
162672024-02-27T21:50:20.145Z 4 080-099 146 146 146 267 267 267 T T T
162682024-02-27T21:50:20.145Z 5 100-119 149 149 149 271 271 271 T T T
162692024-02-27T21:50:20.146Z 6 120-139 149 149 149 271 271 271 F F F
162702024-02-27T21:50:20.146Z 7 140-159 146 146 146 267 267 267 F T T <---
162712024-02-27T21:50:20.146Z 8 160-179 149 149 149 271 271 271 F F F
162722024-02-27T21:50:20.146Z 9 180-199 148 148 148 270 270 270 F T T <---
162732024-02-27T21:50:20.146Z 10 200-219 148 148 148 270 270 270 F T T <---
162742024-02-27T21:50:20.146Z 11 220-239 146 146 146 266 266 266 F F F
162752024-02-27T21:50:20.146Z 12 240-259 149 149 149 271 271 271 F F F
162762024-02-27T21:50:20.146Z 13 260-279 146 146 146 267 267 267 F T T <---
162772024-02-27T21:50:20.149Z 14 280-299 144 144 144 264 264 264 F T T <---
162782024-02-27T21:50:20.149Z 15 300-319 146 146 146 266 266 266 F T T <---
162792024-02-27T21:50:20.149Z 16 320-339 146 146 146 266 266 266 F T T <---
162802024-02-27T21:50:20.149Z 17 340-359 145 145 145 265 265 265 F T T <---
162812024-02-27T21:50:20.149Z 18 360-379 145 145 145 265 265 265 F F F
162822024-02-27T21:50:20.149Z 19 380-399 147 147 147 268 268 268 F T T <---
162832024-02-27T21:50:20.150Z 20 400-419 146 146 146 267 267 267 F F F
162842024-02-27T21:50:20.150Z 21 420-439 144 144 144 263 263 263 F F F
162852024-02-27T21:50:20.150Z 22 440-459 148 148 148 269 269 269 F F F
162862024-02-27T21:50:20.150Z 23 460-479 141 141 141 258 258 258 F F F
162872024-02-27T21:50:20.150Z 24 480-499 148 148 148 269 269 269 F F F
162882024-02-27T21:50:20.150Z 25 500-519 148 148 148 269 269 269 F F F
162892024-02-27T21:50:20.150Z 26 520-539 149 149 149 271 271 271 F F F
162902024-02-27T21:50:20.150Z 27 540-559 147 147 147 268 268 268 F T T <---
162912024-02-27T21:50:20.150Z 28 560-579 148 148 148 270 270 270 F F F
162922024-02-27T21:50:20.150Z 29 580-599 149 149 149 271 271 271 F F F
162932024-02-27T21:50:20.150ZMax gen: 149, Max flush: 271
162942024-02-27T21:50:20.150ZError: Difference in extent metadata found!
162952024-02-27T21:50:20.150ZA Difference in extent metadata is expected here
162962024-02-27T21:50:20.150ZOn loop 75
162972024-02-27T21:50:20.150Z
162982024-02-27T21:50:20.154ZVerifying data now
162992024-02-27T21:50:25.887ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
163002024-02-27T21:50:25.890Z95c95
163012024-02-27T21:50:25.890Z< 8,
163022024-02-27T21:50:25.890Z---
163032024-02-27T21:50:25.890Z> 9,
163042024-02-27T21:50:25.890Zdiff found after verify
163052024-02-27T21:50:25.893ZLoop: 75 Downstairs dump after verify (and repair):
163062024-02-27T21:50:25.908ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163072024-02-27T21:50:25.908Z 0 000-019 147 147 147 268 268 268 F F F
163082024-02-27T21:50:25.908Z 1 020-039 148 148 148 269 269 269 F F F
163092024-02-27T21:50:25.908Z 2 040-059 150 150 150 272 272 272 F F F
163102024-02-27T21:50:25.908Z 3 060-079 150 150 150 272 272 272 F F F
163112024-02-27T21:50:25.908Z 4 080-099 150 150 150 272 272 272 F F F
163122024-02-27T21:50:25.908Z 5 100-119 150 150 150 272 272 272 F F F
163132024-02-27T21:50:25.908Z 6 120-139 149 149 149 271 271 271 F F F
163142024-02-27T21:50:25.908Z 7 140-159 150 150 150 272 272 272 F F F
163152024-02-27T21:50:25.908Z 8 160-179 149 149 149 271 271 271 F F F
163162024-02-27T21:50:25.908Z 9 180-199 150 150 150 272 272 272 F F F
163172024-02-27T21:50:25.908Z 10 200-219 150 150 150 272 272 272 F F F
163182024-02-27T21:50:25.912Z 11 220-239 146 146 146 266 266 266 F F F
163192024-02-27T21:50:25.912Z 12 240-259 149 149 149 271 271 271 F F F
163202024-02-27T21:50:25.912Z 13 260-279 150 150 150 272 272 272 F F F
163212024-02-27T21:50:25.912Z 14 280-299 150 150 150 272 272 272 F F F
163222024-02-27T21:50:25.912Z 15 300-319 150 150 150 272 272 272 F F F
163232024-02-27T21:50:25.912Z 16 320-339 150 150 150 272 272 272 F F F
163242024-02-27T21:50:25.912Z 17 340-359 150 150 150 272 272 272 F F F
163252024-02-27T21:50:25.912Z 18 360-379 145 145 145 265 265 265 F F F
163262024-02-27T21:50:25.912Z 19 380-399 150 150 150 272 272 272 F F F
163272024-02-27T21:50:25.912Z 20 400-419 146 146 146 267 267 267 F F F
163282024-02-27T21:50:25.912Z 21 420-439 144 144 144 263 263 263 F F F
163292024-02-27T21:50:25.912Z 22 440-459 148 148 148 269 269 269 F F F
163302024-02-27T21:50:25.912Z 23 460-479 141 141 141 258 258 258 F F F
163312024-02-27T21:50:25.912Z 24 480-499 148 148 148 269 269 269 F F F
163322024-02-27T21:50:25.912Z 25 500-519 148 148 148 269 269 269 F F F
163332024-02-27T21:50:25.912Z 26 520-539 149 149 149 271 271 271 F F F
163342024-02-27T21:50:25.912Z 27 540-559 150 150 150 272 272 272 F F F
163352024-02-27T21:50:25.913Z 28 560-579 148 148 148 270 270 270 F F F
163362024-02-27T21:50:25.913Z 29 580-599 149 149 149 271 271 271 F F F
163372024-02-27T21:50:25.913ZMax gen: 150, Max flush: 272
163382024-02-27T21:50:25.913Z
163392024-02-27T21:50:25.913ZBegin loop 76
163402024-02-27T21:50:25.913ZDownstairs to restart: 2
163412024-02-27T21:50:25.931Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.295737377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758}
163422024-02-27T21:50:25.931Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296075517Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758}
163432024-02-27T21:50:25.938Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296097944Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758}
163442024-02-27T21:50:25.939Z{"msg":"Crucible 5a90e613-593b-427b-bb66-7311821a0e46 has session id: b00d201c-76d2-486b-a10e-1961643ae5b5","v":0,"name":"crucible","level":30Crucible runtime is spawned
163452024-02-27T21:50:25.939Z,"time":"2024-02-27T21:50:24.29613881Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163462024-02-27T21:50:25.939Z{"msg":"Upstairs opts: Upstairs UUID: 5a90e613-593b-427b-bb66-7311821a0e46, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296194973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163472024-02-27T21:50:25.939Z{"msg":"Crucible stats registered with UUID: 5a90e613-593b-427b-bb66-7311821a0e46","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296229729Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163482024-02-27T21:50:25.939Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296254976Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758}
163492024-02-27T21:50:25.939Z{"msg":"5a90e613-593b-427b-bb66-7311821a0e46 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296351165Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163502024-02-27T21:50:25.939Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296379062Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163512024-02-27T21:50:25.939Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296401149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163522024-02-27T21:50:25.939Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296415777Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163532024-02-27T21:50:25.939Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296452363Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163542024-02-27T21:50:25.939Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296489578Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163552024-02-27T21:50:25.939Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296516235Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163562024-02-27T21:50:25.939Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296548251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163572024-02-27T21:50:25.939Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296613404Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163582024-02-27T21:50:25.939Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.29665099Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163592024-02-27T21:50:25.939Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296759087Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163602024-02-27T21:50:25.939Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296846836Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163612024-02-27T21:50:25.939Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.296955414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163622024-02-27T21:50:25.939Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.297464114Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163632024-02-27T21:50:25.939Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.297503409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163642024-02-27T21:50:25.939Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.297560352Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163652024-02-27T21:50:25.939Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.297657671Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163662024-02-27T21:50:25.940Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.297678928Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163672024-02-27T21:50:25.940Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.297705695Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163682024-02-27T21:50:25.940Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.298207876Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163692024-02-27T21:50:25.940Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.298371897Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163702024-02-27T21:50:25.940Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.298413332Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163712024-02-27T21:50:25.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.298734094Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163722024-02-27T21:50:25.940Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.298774049Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163732024-02-27T21:50:25.940Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.298810055Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163742024-02-27T21:50:25.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.298965557Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163752024-02-27T21:50:25.940Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.29902544Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163762024-02-27T21:50:25.940Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299071044Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163772024-02-27T21:50:25.940Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.29910783Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163782024-02-27T21:50:25.940Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299141486Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163792024-02-27T21:50:25.940Z{"msg":"[0]R flush_numbers[0..12]: [268, 269, 272, 272, 272, 272, 271, 272, 271, 272, 272, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299177312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163802024-02-27T21:50:25.940Z{"msg":"[0]R generation[0..12]: [147, 148, 150, 150, 150, 150, 149, 150, 149, 150, 150, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299212158Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163812024-02-27T21:50:25.940Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299248633Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163822024-02-27T21:50:25.940Z{"msg":"[1]R flush_numbers[0..12]: [268, 269, 272, 272, 272, 272, 271, 272, 271, 272, 272, 266]","v":0,"Wait for a query_work_queue command to finish before sending IO
163832024-02-27T21:50:25.940Zname":"crucible","level":30,"time":"2024-02-27T21:50:24.299289179Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163842024-02-27T21:50:25.940Z{"msg":"[1]R generation[0..12]: [147, 148, 150, 150, 150, 150, 149, 150, 149, 150, 150, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299329434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163852024-02-27T21:50:25.940Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299370049Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163862024-02-27T21:50:25.940Z{"msg":"[2]R flush_numbers[0..12]: [268, 269, 272, 272, 272, 272, 271, 272, 271, 272, 272, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299409454Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163872024-02-27T21:50:25.940Z{"msg":"[2]R generation[0..12]: [147, 148, 150, 150, 150, 150, 149, 150, 149, 150, 150, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.2994497Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163882024-02-27T21:50:25.941Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299488155Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163892024-02-27T21:50:25.941Z{"msg":"Max found gen is 151","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.29953203Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163902024-02-27T21:50:25.941Z{"msg":"Generation requested: 152 >= found:151","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299564966Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163912024-02-27T21:50:25.941Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299598622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163922024-02-27T21:50:25.941Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299635568Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"":"downstairs","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163932024-02-27T21:50:25.941Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299663984Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163942024-02-27T21:50:25.941Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299688812Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163952024-02-27T21:50:25.941Z0","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163962024-02-27T21:50:25.941Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299731027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"1","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163972024-02-27T21:50:25.941Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299747895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"client":"2","":"downstairs","session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163982024-02-27T21:50:25.941Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299771542Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
163992024-02-27T21:50:25.941Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299797909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
164002024-02-27T21:50:25.941Z{"msg":"5a90e613-593b-427b-bb66-7311821a0e46 is now active with session: b00d201c-76d2-486b-a10e-1961643ae5b5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299819406Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
164012024-02-27T21:50:25.942Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299831925Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758,"session_id":"b00d201c-76d2-486b-a10e-1961643ae5b5"}
164022024-02-27T21:50:25.942Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:24.299850682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758}
164032024-02-27T21:50:25.942ZLoading write count information from file "/tmp/test_repair_verify.data"
164042024-02-27T21:50:25.942ZRun Repair workload
164052024-02-27T21:50:25.942Z01/30 Write block 243 len 4096 data: 8
164062024-02-27T21:50:26.362Z02/30 Write block 243 len 32768 data: 9 11 12 13 13 12 13 13
164072024-02-27T21:50:26.362Z03/30 Write block 295 len 12288 data: 18 16 16
164082024-02-27T21:50:26.362Z04/30 Read block 4 len 20480
164092024-02-27T21:50:26.362Z05/30 Read block 125 len 12288
164102024-02-27T21:50:26.362Z06/30 Read block 112 len 32768
164112024-02-27T21:50:26.362Z07/30 Write block 18 len 16384 data: 15 14 14 13
164122024-02-27T21:50:26.362Z08/30 Read block 580 len 40960
164132024-02-27T21:50:26.362Z09/30 Read block 449 len 32768
164142024-02-27T21:50:26.362Z10/30 Flush
164152024-02-27T21:50:26.363Z11/30 Write block 323 len 20480 data: 12 11 9 7 10
164162024-02-27T21:50:26.363Z12/30 Read block 355 len 8192
164172024-02-27T21:50:26.363Z13/30 Write block 564 len 20480 data: 12 10 8 8 10
164182024-02-27T21:50:26.363Z14/30 Write block 572 len 12288 data: 10 11 9
164192024-02-27T21:50:26.368Z15/30 Read block 486 len 12288
164202024-02-27T21:50:26.368Z16/30 Read block 560 len 36864
164212024-02-27T21:50:26.368Z17/30 Read block 143 len 24576
164222024-02-27T21:50:26.368Z18/30 Read block 564 len 24576
164232024-02-27T21:50:26.368Z19/30 Read block 101 len 16384
164242024-02-27T21:50:26.368Z20/30 Write block 551 len 24576 data: 17 14 15 16 15 13
164252024-02-27T21:50:26.368Z21/30 Write block 537 len 28672 data: 9 7 7 8 8 9 10
164262024-02-27T21:50:26.368Z22/30 Write block 303 len 4096 data: 12
164272024-02-27T21:50:26.368Z23/30 Write block 69 len 40960 data: 9 9 9 12 9 10 11 13 13 10
164282024-02-27T21:50:26.368Z24/30 Read block 256 len 40960
164292024-02-27T21:50:26.368Z25/30 Write block 538 len 8192 data: 8 8
164302024-02-27T21:50:26.368Z26/30 Read block 409 len 12288
164312024-02-27T21:50:26.369Z27/30 Read block 438 len 36864
164322024-02-27T21:50:26.369Z28/30 Read block 172 len 40960
164332024-02-27T21:50:26.369Z29/30 Write block 210 len 24576 data: 12 12 13 11 13 15
164342024-02-27T21:50:26.369Z30/30 Read block 384 len 20480
164352024-02-27T21:50:26.369Z----------------------------------------------------------------
164362024-02-27T21:50:26.369Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
164372024-02-27T21:50:26.369ZStates: Active Active Active
164382024-02-27T21:50:26.369ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164392024-02-27T21:50:26.369Z 1 Acked 1000 Write 1 Done Done Done false
164402024-02-27T21:50:26.369Z 2 Acked 1001 Write 8 Done Done Done false
164412024-02-27T21:50:26.369Z 3 Acked 1002 Write 3 Done Done Sent false
164422024-02-27T21:50:26.369Z 4 Acked 1003 Read 5 Done Done Sent false
164432024-02-27T21:50:26.369Z 5 Acked 1004 Read 3 Done Done Sent false
164442024-02-27T21:50:26.369Z 6 Acked 1005 Read 8 Done Done Sent false
164452024-02-27T21:50:26.369Z 7 Acked 1006 Write 4 Done Done Sent false
164462024-02-27T21:50:26.369Z 8 Acked 1007 Read 10 Done Done Sent false
164472024-02-27T21:50:26.369Z 9 Acked 1008 Read 8 Done Done Sent false
164482024-02-27T21:50:26.369Z 10 Acked 1009 Flush 0 Done Done Sent false
164492024-02-27T21:50:26.369Z 11 Acked 1010 Write 5 Done Done Sent false
164502024-02-27T21:50:26.369Z 12 Acked 1011 Read 2 Done Done Sent false
164512024-02-27T21:50:26.369Z 13 Acked 1012 Write 5 Done Done Sent false
164522024-02-27T21:50:26.369Z 14 Acked 1013 Write 3 Done Done Sent false
164532024-02-27T21:50:26.369Z 15 Acked 1014 Read 3 Done Done Sent false
164542024-02-27T21:50:26.369Z 16 Acked 1015 Read 9 Done Done Sent false
164552024-02-27T21:50:26.369Z 17 Acked 1016 Read 6 Done Done Sent false
164562024-02-27T21:50:26.370Z 18 Acked 1017 Read 6 Done Done Sent false
164572024-02-27T21:50:26.370Z 19 Acked 1018 Read 4 Done Done Sent false
164582024-02-27T21:50:26.370Z 20 Acked 1019 Write 6 Done Done Sent false
164592024-02-27T21:50:26.370Z 21 Acked 1020 Write 7 Done Done Sent false
164602024-02-27T21:50:26.370Z 22 Acked 1021 Write 1 Done Done Sent false
164612024-02-27T21:50:26.370Z 23 Acked 1022 Write 10 Done Done Sent false
164622024-02-27T21:50:26.370Z 24 Acked 1023 Read 10 Done Done Sent false
164632024-02-27T21:50:26.370Z 25 Acked 1024 Write 2 Done Done Sent false
164642024-02-27T21:50:26.370Z 26 Acked 1025 Read 3 Done Done Sent false
164652024-02-27T21:50:26.370Z 27 Acked 1026 Read 9 Done Done Sent false
164662024-02-27T21:50:26.370Z 28 Acked 1027 Read 10 Done Done Sent false
164672024-02-27T21:50:26.370Z 29 Acked 1028 Write 6 Done Done Sent false
164682024-02-27T21:50:26.370Z 30 Acked 1029 Read 5 Done Sent Sent false
164692024-02-27T21:50:26.370Z STATES DS:0 DS:1 DS:2 TOTAL
164702024-02-27T21:50:26.370Z New 0 0 0 0
164712024-02-27T21:50:26.370Z Sent 0 1 28 29
164722024-02-27T21:50:26.370Z Done 30 29 2 61
164732024-02-27T21:50:26.370Z Skipped 0 0 0 0
164742024-02-27T21:50:26.370Z Error 0 0 0 0
164752024-02-27T21:50:26.370ZLast Flush: 1009 1009 0
164762024-02-27T21:50:26.370ZDownstairs last five completed:
164772024-02-27T21:50:26.370ZUpstairs last five completed: 30 29 28 27 26
164782024-02-27T21:50:26.370Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:24.327945363Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1758}
164792024-02-27T21:50:26.370ZWrote out file "/tmp/test_repair_verify.data"
164802024-02-27T21:50:26.371Z
164812024-02-27T21:50:26.371ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164822024-02-27T21:50:26.371ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164832024-02-27T21:50:26.371Z 0 000-019 152 152 147 273 273 268 F F F <---
164842024-02-27T21:50:26.371Z 1 020-039 152 152 148 273 273 269 F F F <---
164852024-02-27T21:50:26.371Z 2 040-059 150 150 150 272 272 272 F F F
164862024-02-27T21:50:26.371Z 3 060-079 150 150 150 272 272 272 T T F <---
164872024-02-27T21:50:26.371Z 4 080-099 150 150 150 272 272 272 F F F
164882024-02-27T21:50:26.371Z 5 100-119 150 150 150 272 272 272 F F F
164892024-02-27T21:50:26.371Z 6 120-139 149 149 149 271 271 271 F F F
164902024-02-27T21:50:26.371Z 7 140-159 150 150 150 272 272 272 F F F
164912024-02-27T21:50:26.371Z 8 160-179 149 149 149 271 271 271 F F F
164922024-02-27T21:50:26.371Z 9 180-199 150 150 150 272 272 272 F F F
164932024-02-27T21:50:26.371Z 10 200-219 150 150 150 272 272 272 T T F <---
164942024-02-27T21:50:26.371Z 11 220-239 146 146 146 266 266 266 F F F
164952024-02-27T21:50:26.371Z 12 240-259 152 152 149 273 273 271 F F T <---
164962024-02-27T21:50:26.371Z 13 260-279 150 150 150 272 272 272 F F F
164972024-02-27T21:50:26.371Z 14 280-299 152 152 150 273 273 272 F F F <---
164982024-02-27T21:50:26.371Z 15 300-319 150 150 150 272 272 272 T T F <---
164992024-02-27T21:50:26.371Z 16 320-339 150 150 150 272 272 272 T T F <---
165002024-02-27T21:50:26.371Z 17 340-359 150 150 150 272 272 272 F F F
165012024-02-27T21:50:26.371Z 18 360-379 145 145 145 265 265 265 F F F
165022024-02-27T21:50:26.371Z 19 380-399 150 150 150 272 272 272 F F F
165032024-02-27T21:50:26.371Z 20 400-419 146 146 146 267 267 267 F F F
165042024-02-27T21:50:26.371Z 21 420-439 144 144 144 263 263 263 F F F
165052024-02-27T21:50:26.371Z 22 440-459 148 148 148 269 269 269 F F F
165062024-02-27T21:50:26.372Z 23 460-479 141 141 141 258 258 258 F F F
165072024-02-27T21:50:26.372Z 24 480-499 148 148 148 269 269 269 F F F
165082024-02-27T21:50:26.372Z 25 500-519 148 148 148 269 269 269 F F F
165092024-02-27T21:50:26.372Z 26 520-539 149 149 149 271 271 271 T T F <---
165102024-02-27T21:50:26.372Z 27 540-559 150 150 150 272 272 272 T T F <---
165112024-02-27T21:50:26.372Z 28 560-579 148 148 148 270 270 270 T T F <---
165122024-02-27T21:50:26.372Z 29 580-599 149 149 149 271 271 271 F F F
165132024-02-27T21:50:26.372ZMax gen: 152, Max flush: 273
165142024-02-27T21:50:26.372ZError: Difference in extent metadata found!
165152024-02-27T21:50:26.372ZA Difference in extent metadata is expected here
165162024-02-27T21:50:26.372ZOn loop 76
165172024-02-27T21:50:26.372Z
165182024-02-27T21:50:26.372ZVerifying data now
165192024-02-27T21:50:30.564ZNo change after verify
165202024-02-27T21:50:30.564ZLoop: 76 Downstairs dump after verify (and repair):
165212024-02-27T21:50:30.578ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165222024-02-27T21:50:30.578Z 0 000-019 152 152 152 273 273 273 F F F
165232024-02-27T21:50:30.578Z 1 020-039 152 152 152 273 273 273 F F F
165242024-02-27T21:50:30.578Z 2 040-059 150 150 150 272 272 272 F F F
165252024-02-27T21:50:30.579Z 3 060-079 153 153 153 274 274 274 F F F
165262024-02-27T21:50:30.579Z 4 080-099 150 150 150 272 272 272 F F F
165272024-02-27T21:50:30.579Z 5 100-119 150 150 150 272 272 272 F F F
165282024-02-27T21:50:30.579Z 6 120-139 149 149 149 271 271 271 F F F
165292024-02-27T21:50:30.579Z 7 140-159 150 150 150 272 272 272 F F F
165302024-02-27T21:50:30.579Z 8 160-179 149 149 149 271 271 271 F F F
165312024-02-27T21:50:30.579Z 9 180-199 150 150 150 272 272 272 F F F
165322024-02-27T21:50:30.579Z 10 200-219 153 153 153 274 274 274 F F F
165332024-02-27T21:50:30.579Z 11 220-239 146 146 146 266 266 266 F F F
165342024-02-27T21:50:30.579Z 12 240-259 152 152 152 273 273 273 F F F
165352024-02-27T21:50:30.584Z 13 260-279 150 150 150 272 272 272 F F F
165362024-02-27T21:50:30.584Z 14 280-299 152 152 152 273 273 273 F F F
165372024-02-27T21:50:30.584Z 15 300-319 153 153 153 274 274 274 F F F
165382024-02-27T21:50:30.584Z 16 320-339 153 153 153 274 274 274 F F F
165392024-02-27T21:50:30.584Z 17 340-359 150 150 150 272 272 272 F F F
165402024-02-27T21:50:30.584Z 18 360-379 145 145 145 265 265 265 F F F
165412024-02-27T21:50:30.584Z 19 380-399 150 150 150 272 272 272 F F F
165422024-02-27T21:50:30.584Z 20 400-419 146 146 146 267 267 267 F F F
165432024-02-27T21:50:30.584Z 21 420-439 144 144 144 263 263 263 F F F
165442024-02-27T21:50:30.584Z 22 440-459 148 148 148 269 269 269 F F F
165452024-02-27T21:50:30.584Z 23 460-479 141 141 141 258 258 258 F F F
165462024-02-27T21:50:30.584Z 24 480-499 148 148 148 269 269 269 F F F
165472024-02-27T21:50:30.584Z 25 500-519 148 148 148 269 269 269 F F F
165482024-02-27T21:50:30.584Z 26 520-539 153 153 153 274 274 274 F F F
165492024-02-27T21:50:30.584Z 27 540-559 153 153 153 274 274 274 F F F
165502024-02-27T21:50:30.584Z 28 560-579 153 153 153 274 274 274 F F F
165512024-02-27T21:50:30.584Z 29 580-599 149 149 149 271 271 271 F F F
165522024-02-27T21:50:30.584ZMax gen: 153, Max flush: 274
165532024-02-27T21:50:30.584Z
165542024-02-27T21:50:30.584ZBegin loop 77
165552024-02-27T21:50:30.584ZDownstairs to restart: 1
165562024-02-27T21:50:30.601Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.965432172Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766}
165572024-02-27T21:50:30.601Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.965720378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766}
165582024-02-27T21:50:30.601Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name"Crucible runtime is spawned
165592024-02-27T21:50:30.607Z:"crucible","level":30,"time":"2024-02-27T21:50:28.965757674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766}
165602024-02-27T21:50:30.607Z{"msg":"Crucible ec6832be-59c1-4de9-928b-871da734d84c has session id: 93eb3d4c-1ef0-4a83-a96d-3f360820efc1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.965797019Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165612024-02-27T21:50:30.607Z{"msg":"Upstairs opts: Upstairs UUID: ec6832be-59c1-4de9-928b-871da734d84c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.965841144Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165622024-02-27T21:50:30.608Z{"msg":"Crucible stats registered with UUID: ec6832be-59c1-4de9-928b-871da734d84c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.965885239Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165632024-02-27T21:50:30.608Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.965913556Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766}
165642024-02-27T21:50:30.608Z{"msg":"ec6832be-59c1-4de9-928b-871da734d84c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966112982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165652024-02-27T21:50:30.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966165886Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165662024-02-27T21:50:30.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.96621666Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165672024-02-27T21:50:30.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966265034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165682024-02-27T21:50:30.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966316938Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165692024-02-27T21:50:30.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966351434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165702024-02-27T21:50:30.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966381191Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165712024-02-27T21:50:30.608Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966425685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165722024-02-27T21:50:30.608Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966496517Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165732024-02-27T21:50:30.608Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966549611Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165742024-02-27T21:50:30.608Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966591936Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165752024-02-27T21:50:30.608Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.96664303Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165762024-02-27T21:50:30.608Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.966674996Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165772024-02-27T21:50:30.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.967221972Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165782024-02-27T21:50:30.608Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.967264356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165792024-02-27T21:50:30.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.967297892Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165802024-02-27T21:50:30.608Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.967324909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165812024-02-27T21:50:30.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.967350536Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165822024-02-27T21:50:30.608Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.967372304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165832024-02-27T21:50:30.608Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.96782782Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165842024-02-27T21:50:30.608Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.967861236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165852024-02-27T21:50:30.608Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.967879044Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165862024-02-27T21:50:30.609Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968413331Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165872024-02-27T21:50:30.609Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968464825Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165882024-02-27T21:50:30.609Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968485063Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165892024-02-27T21:50:30.609Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.96850694Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165902024-02-27T21:50:30.609Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968521028Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165912024-02-27T21:50:30.609Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968542846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165922024-02-27T21:50:30.609Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968555894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165932024-02-27T21:50:30.609Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968575892Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165942024-02-27T21:50:30.609Z{"msg":"[0]R flush_numbers[0..12]: [273, 273, 272, 274, 272, 272, 271, 272, 271, 272, 274, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.9685904Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165952024-02-27T21:50:30.609Z{"msg":"[0]R generation[0..12]: [152, 152, 150, 153, 150, 150, 149, 150, 149, 150, 153, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968610928Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165962024-02-27T21:50:30.609Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968625586Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165972024-02-27T21:50:30.609Z{"msg":"[1]R flush_numbers[0..12]: [273, 273, 272, 274, 272, 272, 271, 272, 271, 272, 274, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968646144Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165982024-02-27T21:50:30.609Z{"msg":"[1]R generation[0..12]: [152, 152, 150, 153, 150, 150, 149, 150, 149, 150, 153, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968660162Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
165992024-02-27T21:50:30.609Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.96867814Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166002024-02-27T21:50:30.609Z{"msg":"[2]R flush_numbers[0..12]: [273, 273, 272, 274, 272, 272, 271, 272, 271, 272, 274, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968693838Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166012024-02-27T21:50:30.609Z{"msg":"[2]R generation[0..12]: [152, 152, 150, 153, 150, 150, 149, 150, 149, 150, 153, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968708946Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166022024-02-27T21:50:30.609Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
166032024-02-27T21:50:30.609Z,"time":"2024-02-27T21:50:28.968723135Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166042024-02-27T21:50:30.609Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968739923Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166052024-02-27T21:50:30.609Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.96876084Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166062024-02-27T21:50:30.609Z{"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968791976Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166072024-02-27T21:50:30.609Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968808095Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"":"downstairs","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166082024-02-27T21:50:30.609Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968841781Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166092024-02-27T21:50:30.609Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968876837Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"0","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166102024-02-27T21:50:30.609Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968911913Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"1","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166112024-02-27T21:50:30.610Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968943409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"client":"2","":"downstairs","session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166122024-02-27T21:50:30.610Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968969366Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166132024-02-27T21:50:30.610Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.968983164Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166142024-02-27T21:50:30.610Z{"msg":"ec6832be-59c1-4de9-928b-871da734d84c is now active with session: 93eb3d4c-1ef0-4a83-a96d-3f360820efc1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.969003572Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166152024-02-27T21:50:30.610Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.969034578Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766,"session_id":"93eb3d4c-1ef0-4a83-a96d-3f360820efc1"}
166162024-02-27T21:50:30.610Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:28.969050986Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766}
166172024-02-27T21:50:30.610ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
166182024-02-27T21:50:30.613ZLoading write count information from file "/tmp/test_repair_verify.data"
166192024-02-27T21:50:30.613ZRun Repair workload
166202024-02-27T21:50:30.613Z01/30 Write block 263 len 12288 data: 12 14 14
166212024-02-27T21:50:30.613Z02/30 Write block 520 len 32768 data: 13 10 11 9 10 9 10 10
166222024-02-27T21:50:30.613Z03/30 Read block 170 len 32768
166232024-02-27T21:50:30.613Z04/30 Read block 285 len 28672
166242024-02-27T21:50:30.613Z05/30 Read block 456 len 36864
166252024-02-27T21:50:30.613Z06/30 Write block 235 len 4096 data: 18
166262024-02-27T21:50:30.613Z07/30 Flush
166272024-02-27T21:50:30.616Z08/30 Write block 227 len 20480 data: 11 12 11 14 15
166282024-02-27T21:50:30.616Z09/30 Write block 168 len 36864 data: 6 8 9 9 9 8 12 11 13
166292024-02-27T21:50:30.616Z10/30 Read block 129 len 8192
166302024-02-27T21:50:30.623Z11/30 Read block 375 len 8192
166312024-02-27T21:50:30.623Z12/30 Read block 564 len 8192
166322024-02-27T21:50:30.623Z13/30 Read block 554 len 28672
166332024-02-27T21:50:30.626Z14/30 Write block 548 len 24576 data: 15 17 16 18 15 16
166342024-02-27T21:50:30.626Z15/30 Read block 525 len 24576
166352024-02-27T21:50:30.626Z16/30 Flush
166362024-02-27T21:50:30.626Z17/30 Write block 428 len 4096 data: 8
166372024-02-27T21:50:30.626Z18/30 Write block 564 len 4096 data: 13
166382024-02-27T21:50:30.626Z19/30 Read block 352 len 16384
166392024-02-27T21:50:30.626Z20/30 Read block 364 len 40960
166402024-02-27T21:50:30.629Z21/30 Write block 251 len 8192 data: 14 14
166412024-02-27T21:50:30.629Z22/30 Write block 225 len 28672 data: 9 10 12 13 12 15 16
166422024-02-27T21:50:30.629Z23/30 Write block 553 len 20480 data: 17 17 16 14 19
166432024-02-27T21:50:30.629Z24/30 Read block 165 len 28672
166442024-02-27T21:50:30.633Z25/30 Read block 140 len 28672
166452024-02-27T21:50:30.633Z26/30 Read block 325 len 24576
166462024-02-27T21:50:30.633Z27/30 Read block 563 len 16384
166472024-02-27T21:50:30.637Z28/30 Read block 371 len 24576
166482024-02-27T21:50:30.638Z29/30 Write block 400 len 20480 data: 14 12 10 10 10
166492024-02-27T21:50:30.638Z30/30 Write block 322 len 8192 data: 14 13
166502024-02-27T21:50:30.638Z----------------------------------------------------------------
166512024-02-27T21:50:30.638Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
166522024-02-27T21:50:30.638ZStates: Active Active Active
166532024-02-27T21:50:30.638ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166542024-02-27T21:50:30.638Z 1 Acked 1000 Write 3 Done Sent Done false
166552024-02-27T21:50:30.638Z 2 Acked 1001 Write 8 Done Sent Done false
166562024-02-27T21:50:30.638Z 3 Acked 1002 Read 8 Done Sent Done false
166572024-02-27T21:50:30.638Z 4 Acked 1003 Read 7 Done Sent Done false
166582024-02-27T21:50:30.638Z 5 Acked 1004 Read 9 Done Sent Done false
166592024-02-27T21:50:30.638Z 6 Acked 1005 Write 1 Done Sent Done false
166602024-02-27T21:50:30.638Z 7 Acked 1006 Flush 0 Done Sent Done false
166612024-02-27T21:50:30.638Z 8 Acked 1007 Write 5 Done Sent Done false
166622024-02-27T21:50:30.638Z 9 Acked 1008 Write 9 Done Sent Done false
166632024-02-27T21:50:30.638Z 10 Acked 1009 Read 2 Done Sent Done false
166642024-02-27T21:50:30.638Z 11 Acked 1010 Read 2 Done Sent Done false
166652024-02-27T21:50:30.638Z 12 Acked 1011 Read 2 Done Sent Done false
166662024-02-27T21:50:30.638Z 13 Acked 1012 Read 7 Done Sent Done false
166672024-02-27T21:50:30.638Z 14 Acked 1013 Write 6 Done Sent Done false
166682024-02-27T21:50:30.638Z 15 Acked 1014 Read 6 Done Sent Done false
166692024-02-27T21:50:30.638Z 16 Acked 1015 Flush 0 Done Sent Done false
166702024-02-27T21:50:30.639Z 17 Acked 1016 Write 1 Done Sent Done false
166712024-02-27T21:50:30.639Z 18 Acked 1017 Write 1 Done Sent Done false
166722024-02-27T21:50:30.639Z 19 Acked 1018 Read 4 Done Sent Done false
166732024-02-27T21:50:30.639Z 20 Acked 1019 Read 10 Done Sent Done false
166742024-02-27T21:50:30.639Z 21 Acked 1020 Write 2 Done Sent Done false
166752024-02-27T21:50:30.639Z 22 Acked 1021 Write 7 Done Sent Done false
166762024-02-27T21:50:30.639Z 23 Acked 1022 Write 5 Done Sent Done false
166772024-02-27T21:50:30.639Z 24 Acked 1023 Read 7 Done Sent Done false
166782024-02-27T21:50:30.639Z 25 Acked 1024 Read 7 Done Sent Done false
166792024-02-27T21:50:30.639Z 26 Acked 1025 Read 6 Done Sent Done false
166802024-02-27T21:50:30.639Z 27 Acked 1026 Read 4 Done Sent Done false
166812024-02-27T21:50:30.639Z 28 Acked 1027 Read 6 Done Sent Done false
166822024-02-27T21:50:30.639Z 29 Acked 1028 Write 5 Sent Sent Sent false
166832024-02-27T21:50:30.639Z 30 Acked 1029 Write 2 Sent Sent Sent false
166842024-02-27T21:50:30.639Z STATES DS:0 DS:1 DS:2 TOTAL
166852024-02-27T21:50:30.639Z New 0 0 0 0
166862024-02-27T21:50:30.639Z Sent 2 30 2 34
166872024-02-27T21:50:30.639Z Done 28 0 28 56
166882024-02-27T21:50:30.639Z Skipped 0 0 0 0
166892024-02-27T21:50:30.639Z Error 0 0 0 0
166902024-02-27T21:50:30.639ZLast Flush: 1015 0 1015
166912024-02-27T21:50:30.639ZDownstairs last five completed:
166922024-02-27T21:50:30.639ZUpstairs last five completed: 30 29 28 27 26
166932024-02-27T21:50:30.639Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:29.000624347Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1766}
166942024-02-27T21:50:30.639ZWrote out file "/tmp/test_repair_verify.data"
166952024-02-27T21:50:30.643Z
166962024-02-27T21:50:30.646ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166972024-02-27T21:50:30.660ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166982024-02-27T21:50:30.660Z 0 000-019 152 152 152 273 273 273 F F F
166992024-02-27T21:50:30.660Z 1 020-039 152 152 152 273 273 273 F F F
167002024-02-27T21:50:30.660Z 2 040-059 150 150 150 272 272 272 F F F
167012024-02-27T21:50:30.660Z 3 060-079 153 153 153 274 274 274 F F F
167022024-02-27T21:50:30.664Z 4 080-099 150 150 150 272 272 272 F F F
167032024-02-27T21:50:30.664Z 5 100-119 150 150 150 272 272 272 F F F
167042024-02-27T21:50:30.664Z 6 120-139 149 149 149 271 271 271 F F F
167052024-02-27T21:50:30.664Z 7 140-159 150 150 150 272 272 272 F F F
167062024-02-27T21:50:30.664Z 8 160-179 154 149 154 276 271 276 F F F <---
167072024-02-27T21:50:30.664Z 9 180-199 150 150 150 272 272 272 F F F
167082024-02-27T21:50:30.664Z 10 200-219 153 153 153 274 274 274 F F F
167092024-02-27T21:50:30.664Z 11 220-239 154 146 154 276 266 276 T F T <---
167102024-02-27T21:50:30.665Z 12 240-259 152 152 152 273 273 273 T F T <---
167112024-02-27T21:50:30.665Z 13 260-279 154 150 154 275 272 275 F F F <---
167122024-02-27T21:50:30.665Z 14 280-299 152 152 152 273 273 273 F F F
167132024-02-27T21:50:30.665Z 15 300-319 153 153 153 274 274 274 F F F
167142024-02-27T21:50:30.665Z 16 320-339 153 153 153 274 274 274 T F T <---
167152024-02-27T21:50:30.665Z 17 340-359 150 150 150 272 272 272 F F F
167162024-02-27T21:50:30.665Z 18 360-379 145 145 145 265 265 265 F F F
167172024-02-27T21:50:30.665Z 19 380-399 150 150 150 272 272 272 F F F
167182024-02-27T21:50:30.665Z 20 400-419 146 146 146 267 267 267 T F T <---
167192024-02-27T21:50:30.665Z 21 420-439 144 144 144 263 263 263 T F T <---
167202024-02-27T21:50:30.665Z 22 440-459 148 148 148 269 269 269 F F F
167212024-02-27T21:50:30.665Z 23 460-479 141 141 141 258 258 258 F F F
167222024-02-27T21:50:30.665Z 24 480-499 148 148 148 269 269 269 F F F
167232024-02-27T21:50:30.665Z 25 500-519 148 148 148 269 269 269 F F F
167242024-02-27T21:50:30.665Z 26 520-539 154 153 154 275 274 275 F F F <---
167252024-02-27T21:50:30.665Z 27 540-559 154 153 154 276 274 276 T F T <---
167262024-02-27T21:50:30.665Z 28 560-579 153 153 153 274 274 274 T F T <---
167272024-02-27T21:50:30.665Z 29 580-599 149 149 149 271 271 271 F F F
167282024-02-27T21:50:30.665ZMax gen: 154, Max flush: 276
167292024-02-27T21:50:30.665ZError: Difference in extent metadata found!
167302024-02-27T21:50:30.665ZA Difference in extent metadata is expected here
167312024-02-27T21:50:30.665ZOn loop 77
167322024-02-27T21:50:30.665Z
167332024-02-27T21:50:30.669ZVerifying data now
167342024-02-27T21:50:31.203ZNo change after verify
167352024-02-27T21:50:31.203ZLoop: 77 Downstairs dump after verify (and repair):
167362024-02-27T21:50:31.211ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167372024-02-27T21:50:31.211Z 0 000-019 152 152 152 273 273 273 F F F
167382024-02-27T21:50:31.211Z 1 020-039 152 152 152 273 273 273 F F F
167392024-02-27T21:50:31.211Z 2 040-059 150 150 150 272 272 272 F F F
167402024-02-27T21:50:31.211Z 3 060-079 153 153 153 274 274 274 F F F
167412024-02-27T21:50:31.211Z 4 080-099 150 150 150 272 272 272 F F F
167422024-02-27T21:50:31.211Z 5 100-119 150 150 150 272 272 272 F F F
167432024-02-27T21:50:31.211Z 6 120-139 149 149 149 271 271 271 F F F
167442024-02-27T21:50:31.215Z 7 140-159 150 150 150 272 272 272 F F F
167452024-02-27T21:50:31.215Z 8 160-179 154 154 154 276 276 276 F F F
167462024-02-27T21:50:31.215Z 9 180-199 150 150 150 272 272 272 F F F
167472024-02-27T21:50:31.215Z 10 200-219 153 153 153 274 274 274 F F F
167482024-02-27T21:50:31.215Z 11 220-239 155 155 155 277 277 277 F F F
167492024-02-27T21:50:31.215Z 12 240-259 155 155 155 277 277 277 F F F
167502024-02-27T21:50:31.215Z 13 260-279 154 154 154 275 275 275 F F F
167512024-02-27T21:50:31.215Z 14 280-299 152 152 152 273 273 273 F F F
167522024-02-27T21:50:31.215Z 15 300-319 153 153 153 274 274 274 F F F
167532024-02-27T21:50:31.215Z 16 320-339 155 155 155 277 277 277 F F F
167542024-02-27T21:50:31.215Z 17 340-359 150 150 150 272 272 272 F F F
167552024-02-27T21:50:31.215Z 18 360-379 145 145 145 265 265 265 F F F
167562024-02-27T21:50:31.215Z 19 380-399 150 150 150 272 272 272 F F F
167572024-02-27T21:50:31.215Z 20 400-419 155 155 155 277 277 277 F F F
167582024-02-27T21:50:31.215Z 21 420-439 155 155 155 277 277 277 F F F
167592024-02-27T21:50:31.215Z 22 440-459 148 148 148 269 269 269 F F F
167602024-02-27T21:50:31.215Z 23 460-479 141 141 141 258 258 258 F F F
167612024-02-27T21:50:31.215Z 24 480-499 148 148 148 269 269 269 F F F
167622024-02-27T21:50:31.215Z 25 500-519 148 148 148 269 269 269 F F F
167632024-02-27T21:50:31.216Z 26 520-539 154 154 154 275 275 275 F F F
167642024-02-27T21:50:31.216Z 27 540-559 155 155 155 277 277 277 F F F
167652024-02-27T21:50:31.216Z 28 560-579 155 155 155 277 277 277 F F F
167662024-02-27T21:50:31.216Z 29 580-599 149 149 149 271 271 271 F F F
167672024-02-27T21:50:31.216ZMax gen: 155, Max flush: 277
167682024-02-27T21:50:31.216Z
167692024-02-27T21:50:31.216ZBegin loop 78
167702024-02-27T21:50:31.216ZDownstairs to restart: 0
167712024-02-27T21:50:31.234Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.598445098Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774}
167722024-02-27T21:50:31.234Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.598746312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774}
167732024-02-27T21:50:31.234Z{"msg":"Upstairs <-> Downstairs Message Version: 5"Crucible runtime is spawned
167742024-02-27T21:50:31.241Z,"v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.598803016Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774}
167752024-02-27T21:50:31.241Z{"msg":"Crucible 800e4cc9-e46f-431c-b3f7-f5ae9dddbc66 has session id: 11c26291-0e5a-4b2b-a72a-c5539c191cfb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.59884973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167762024-02-27T21:50:31.241Z{"msg":"Upstairs opts: Upstairs UUID: 800e4cc9-e46f-431c-b3f7-f5ae9dddbc66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.598908203Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167772024-02-27T21:50:31.241Z{"msg":"Crucible stats registered with UUID: 800e4cc9-e46f-431c-b3f7-f5ae9dddbc66","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.598959527Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167782024-02-27T21:50:31.242Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.598982464Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774}
167792024-02-27T21:50:31.242Z{"msg":"800e4cc9-e46f-431c-b3f7-f5ae9dddbc66 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.5990199Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167802024-02-27T21:50:31.242Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599074114Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167812024-02-27T21:50:31.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599129937Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167822024-02-27T21:50:31.242Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599181071Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167832024-02-27T21:50:31.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599217747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167842024-02-27T21:50:31.242Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599248073Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167852024-02-27T21:50:31.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599300997Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167862024-02-27T21:50:31.242Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599335303Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167872024-02-27T21:50:31.242Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599389296Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167882024-02-27T21:50:31.242Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599426312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167892024-02-27T21:50:31.242Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599476396Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167902024-02-27T21:50:31.242Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599510692Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167912024-02-27T21:50:31.243Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.599542869Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167922024-02-27T21:50:31.243Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.600182513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167932024-02-27T21:50:31.243Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.600225518Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167942024-02-27T21:50:31.243Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.600242136Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167952024-02-27T21:50:31.243Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.60038069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167962024-02-27T21:50:31.243Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.600416675Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167972024-02-27T21:50:31.243Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.600444342Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167982024-02-27T21:50:31.243Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.600863643Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
167992024-02-27T21:50:31.243Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.600907658Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168002024-02-27T21:50:31.243Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.600925316Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168012024-02-27T21:50:31.243Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601410209Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168022024-02-27T21:50:31.243Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601462762Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168032024-02-27T21:50:31.243Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601489859Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168042024-02-27T21:50:31.244Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601520345Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168052024-02-27T21:50:31.244Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601537174Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168062024-02-27T21:50:31.244Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.60156874Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168072024-02-27T21:50:31.244Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601585498Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168082024-02-27T21:50:31.244Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601615884Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168092024-02-27T21:50:31.244Z{"msg":"[0]R flush_numbers[0..12]: [273, 273, 272, 274, 272, 272, 271, 272, 276, 272, 274, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601632492Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168102024-02-27T21:50:31.244Z{"msg":"[0]R generation[0..12]: [152, 152, 150, 153, 150, 150, 149, 150, 154, 150, 153, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601665938Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
168112024-02-27T21:50:31.244Z11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168122024-02-27T21:50:31.244Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601687046Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168132024-02-27T21:50:31.244Z{"msg":"[1]R flush_numbers[0..12]: [273, 273, 272, 274, 272, 272, 271, 272, 276, 272, 274, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601718242Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168142024-02-27T21:50:31.244Z{"msg":"[1]R generation[0..12]: [152, 152, 150, 153, 150, 150, 149, 150, 154, 150, 153, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.60173451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168152024-02-27T21:50:31.244Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601767906Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168162024-02-27T21:50:31.244Z{"msg":"[2]R flush_numbers[0..12]: [273, 273, 272, 274, 272, 272, 271, 272, 276, 272, 274, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601794853Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168172024-02-27T21:50:31.245Z{"msg":"[2]R generation[0..12]: [152, 152, 150, 153, 150, 150, 149, 150, 154, 150, 153, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601827259Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168182024-02-27T21:50:31.245Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601844247Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168192024-02-27T21:50:31.245Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601865305Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168202024-02-27T21:50:31.245Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601880103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168212024-02-27T21:50:31.245Z{"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601893672Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168222024-02-27T21:50:31.245Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601914519Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"":"downstairs","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168232024-02-27T21:50:31.245Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601928848Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168242024-02-27T21:50:31.245Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601941116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"0","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168252024-02-27T21:50:31.245Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601961904Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"1","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168262024-02-27T21:50:31.245Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601975702Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"client":"2","":"downstairs","session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168272024-02-27T21:50:31.245Z{"msg":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
168282024-02-27T21:50:31.245Z"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.601998779Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168292024-02-27T21:50:31.245Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.602021146Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168302024-02-27T21:50:31.246Z{"msg":"800e4cc9-e46f-431c-b3f7-f5ae9dddbc66 is now active with session: 11c26291-0e5a-4b2b-a72a-c5539c191cfb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.602043414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168312024-02-27T21:50:31.246Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.602056273Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774,"session_id":"11c26291-0e5a-4b2b-a72a-c5539c191cfb"}
168322024-02-27T21:50:31.246Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:29.60207554Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774}
168332024-02-27T21:50:31.246ZLoading write count information from file "/tmp/test_repair_verify.data"
168342024-02-27T21:50:31.246ZRun Repair workload
168352024-02-27T21:50:31.252Z01/30 Write block 423 len 40960 data: 10 10 6 6 7 9 9 10 10 10
168362024-02-27T21:50:31.252Z02/30 Read block 563 len 40960
168372024-02-27T21:50:31.253Z03/30 Read block 255 len 28672
168382024-02-27T21:50:31.253Z04/30 Write block 430 len 12288 data: 11 11 11
168392024-02-27T21:50:31.253Z05/30 Write block 374 len 16384 data: 7 7 7 9
168402024-02-27T21:50:31.253Z06/30 Write block 322 len 20480 data: 15 14 12 10 8
168412024-02-27T21:50:31.253Z07/30 Flush
168422024-02-27T21:50:31.253Z08/30 Write block 149 len 20480 data: 11 11 9 9 8
168432024-02-27T21:50:31.253Z09/30 Read block 446 len 36864
168442024-02-27T21:50:31.253Z10/30 Read block 141 len 40960
168452024-02-27T21:50:31.253Z11/30 Write block 217 len 4096 data: 17
168462024-02-27T21:50:31.253Z12/30 Flush
168472024-02-27T21:50:31.257Z13/30 Flush
168482024-02-27T21:50:31.257Z14/30 Write block 553 len 40960 data: 18 18 17 15 20 16 15 16 17 16
168492024-02-27T21:50:31.257Z15/30 Write block 231 len 36864 data: 17 18 17 18 19 19 12 12 12
168502024-02-27T21:50:31.257Z16/30 Write block 23 len 8192 data: 7 12
168512024-02-27T21:50:31.257Z17/30 Read block 224 len 16384
168522024-02-27T21:50:31.257Z18/30 Read block 140 len 24576
168532024-02-27T21:50:31.257Z19/30 Flush
168542024-02-27T21:50:31.262Z20/30 Write block 534 len 4096 data: 10
168552024-02-27T21:50:31.262Z21/30 Write block 385 len 4096 data: 13
168562024-02-27T21:50:31.262Z22/30 Read block 191 len 16384
168572024-02-27T21:50:31.262Z23/30 Read block 538 len 4096
168582024-02-27T21:50:31.262Z24/30 Flush
168592024-02-27T21:50:31.266Z25/30 Write block 70 len 36864 data: 10 10 13 10 11 12 14 14 11
168602024-02-27T21:50:31.266Z26/30 Read block 160 len 28672
168612024-02-27T21:50:31.266Z27/30 Read block 315 len 16384
168622024-02-27T21:50:31.266Z28/30 Write block 108 len 24576 data: 16 14 13 15 13 12
168632024-02-27T21:50:31.269Z29/30 Write block 275 len 32768 data: 8 10 9 8 7 6 5 6
168642024-02-27T21:50:31.270Z30/30 Write block 356 len 4096 data: 15
168652024-02-27T21:50:31.270Z----------------------------------------------------------------
168662024-02-27T21:50:31.270Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:23
168672024-02-27T21:50:31.270ZStates: Active Active Active
168682024-02-27T21:50:31.270ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168692024-02-27T21:50:31.270Z 8 Acked 1007 Write 5 Done Done Done false
168702024-02-27T21:50:31.270Z 9 Acked 1008 Read 9 Sent Done Done false
168712024-02-27T21:50:31.270Z 10 Acked 1009 Read 10 Sent Done Done false
168722024-02-27T21:50:31.270Z 11 Acked 1010 Write 1 Sent Done Done false
168732024-02-27T21:50:31.270Z 12 Acked 1011 Flush 0 Sent Done Done false
168742024-02-27T21:50:31.270Z 13 Acked 1012 Flush 0 Sent Done Done false
168752024-02-27T21:50:31.270Z 14 Acked 1013 Write 10 Sent Done Done false
168762024-02-27T21:50:31.270Z 15 Acked 1014 Write 9 Sent Done Done false
168772024-02-27T21:50:31.270Z 16 Acked 1015 Write 2 Sent Done Done false
168782024-02-27T21:50:31.270Z 17 Acked 1016 Read 4 Sent Done Done false
168792024-02-27T21:50:31.270Z 18 Acked 1017 Read 6 Sent Done Done false
168802024-02-27T21:50:31.271Z 19 Acked 1018 Flush 0 Sent Done Done false
168812024-02-27T21:50:31.271Z 20 Acked 1019 Write 1 Sent Done Done false
168822024-02-27T21:50:31.271Z 21 Acked 1020 Write 1 Sent Done Done false
168832024-02-27T21:50:31.271Z 22 Acked 1021 Read 4 Sent Done Done false
168842024-02-27T21:50:31.271Z 23 Acked 1022 Read 1 Sent Done Done false
168852024-02-27T21:50:31.271Z 24 Acked 1023 Flush 0 Sent Done Done false
168862024-02-27T21:50:31.271Z 25 Acked 1024 Write 9 Sent Done Done false
168872024-02-27T21:50:31.271Z 26 Acked 1025 Read 7 Sent Done Done false
168882024-02-27T21:50:31.271Z 27 Acked 1026 Read 4 Sent Done Done false
168892024-02-27T21:50:31.271Z 28 Acked 1027 Write 6 Sent Done Done false
168902024-02-27T21:50:31.271Z 29 Acked 1028 Write 8 Sent Sent Sent false
168912024-02-27T21:50:31.271Z 30 Acked 1029 Write 1 Sent Sent Sent false
168922024-02-27T21:50:31.271Z STATES DS:0 DS:1 DS:2 TOTAL
168932024-02-27T21:50:31.271Z New 0 0 0 0
168942024-02-27T21:50:31.271Z Sent 22 2 2 26
168952024-02-27T21:50:31.272Z Done 1 21 21 43
168962024-02-27T21:50:31.272Z Skipped 0 0 0 0
168972024-02-27T21:50:31.272Z Error 0 0 0 0
168982024-02-27T21:50:31.272ZLast Flush: 1006 1023 1023
168992024-02-27T21:50:31.272ZDownstairs last five completed: 1006 1005 1004 1003 1002
169002024-02-27T21:50:31.272ZUpstairs last five completed: 30 29 28 27 26
169012024-02-27T21:50:31.272Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:29.63263827Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1774}
169022024-02-27T21:50:31.272ZWrote out file "/tmp/test_repair_verify.data"
169032024-02-27T21:50:31.278Z
169042024-02-27T21:50:31.278ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
169052024-02-27T21:50:31.292ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169062024-02-27T21:50:31.292Z 0 000-019 152 152 152 273 273 273 F F F
169072024-02-27T21:50:31.292Z 1 020-039 152 156 156 273 281 281 F F F <---
169082024-02-27T21:50:31.293Z 2 040-059 150 150 150 272 272 272 F F F
169092024-02-27T21:50:31.293Z 3 060-079 153 153 153 274 274 274 F T T <---
169102024-02-27T21:50:31.293Z 4 080-099 150 150 150 272 272 272 F F F
169112024-02-27T21:50:31.293Z 5 100-119 150 150 150 272 272 272 F T T <---
169122024-02-27T21:50:31.297Z 6 120-139 149 149 149 271 271 271 F F F
169132024-02-27T21:50:31.297Z 7 140-159 150 156 156 272 279 279 T F F <---
169142024-02-27T21:50:31.297Z 8 160-179 154 154 154 276 276 276 F F F
169152024-02-27T21:50:31.297Z 9 180-199 150 150 150 272 272 272 F F F
169162024-02-27T21:50:31.297Z 10 200-219 153 156 156 274 279 279 F F F <---
169172024-02-27T21:50:31.297Z 11 220-239 155 156 156 277 281 281 F F F <---
169182024-02-27T21:50:31.297Z 12 240-259 155 155 155 277 277 277 F F F
169192024-02-27T21:50:31.297Z 13 260-279 154 154 154 275 275 275 F T T <---
169202024-02-27T21:50:31.297Z 14 280-299 152 152 152 273 273 273 F T T <---
169212024-02-27T21:50:31.297Z 15 300-319 153 153 153 274 274 274 F F F
169222024-02-27T21:50:31.298Z 16 320-339 156 156 156 278 278 278 F F F
169232024-02-27T21:50:31.298Z 17 340-359 150 150 150 272 272 272 F T T <---
169242024-02-27T21:50:31.298Z 18 360-379 156 156 156 278 278 278 F F F
169252024-02-27T21:50:31.298Z 19 380-399 150 156 156 272 282 282 F F F <---
169262024-02-27T21:50:31.298Z 20 400-419 155 155 155 277 277 277 F F F
169272024-02-27T21:50:31.298Z 21 420-439 156 156 156 278 278 278 F F F
169282024-02-27T21:50:31.298Z 22 440-459 148 148 148 269 269 269 F F F
169292024-02-27T21:50:31.298Z 23 460-479 141 141 141 258 258 258 F F F
169302024-02-27T21:50:31.298Z 24 480-499 148 148 148 269 269 269 F F F
169312024-02-27T21:50:31.298Z 25 500-519 148 148 148 269 269 269 F F F
169322024-02-27T21:50:31.298Z 26 520-539 154 156 156 275 282 282 F F F <---
169332024-02-27T21:50:31.298Z 27 540-559 155 156 156 277 281 281 F F F <---
169342024-02-27T21:50:31.298Z 28 560-579 155 156 156 277 281 281 F F F <---
169352024-02-27T21:50:31.299Z 29 580-599 149 149 149 271 271 271 F F F
169362024-02-27T21:50:31.299ZMax gen: 156, Max flush: 282
169372024-02-27T21:50:31.299ZError: Difference in extent metadata found!
169382024-02-27T21:50:31.299ZA Difference in extent metadata is expected here
169392024-02-27T21:50:31.299ZOn loop 78
169402024-02-27T21:50:31.299Z
169412024-02-27T21:50:31.304ZVerifying data now
169422024-02-27T21:50:35.949ZNo change after verify
169432024-02-27T21:50:35.949ZLoop: 78 Downstairs dump after verify (and repair):
169442024-02-27T21:50:35.963ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169452024-02-27T21:50:35.963Z 0 000-019 152 152 152 273 273 273 F F F
169462024-02-27T21:50:35.963Z 1 020-039 156 156 156 281 281 281 F F F
169472024-02-27T21:50:35.963Z 2 040-059 150 150 150 272 272 272 F F F
169482024-02-27T21:50:35.963Z 3 060-079 157 157 157 283 283 283 F F F
169492024-02-27T21:50:35.963Z 4 080-099 150 150 150 272 272 272 F F F
169502024-02-27T21:50:35.963Z 5 100-119 157 157 157 283 283 283 F F F
169512024-02-27T21:50:35.966Z 6 120-139 149 149 149 271 271 271 F F F
169522024-02-27T21:50:35.966Z 7 140-159 156 156 156 279 279 279 F F F
169532024-02-27T21:50:35.966Z 8 160-179 154 154 154 276 276 276 F F F
169542024-02-27T21:50:35.966Z 9 180-199 150 150 150 272 272 272 F F F
169552024-02-27T21:50:35.966Z 10 200-219 156 156 156 279 279 279 F F F
169562024-02-27T21:50:35.967Z 11 220-239 156 156 156 281 281 281 F F F
169572024-02-27T21:50:35.967Z 12 240-259 155 155 155 277 277 277 F F F
169582024-02-27T21:50:35.967Z 13 260-279 157 157 157 283 283 283 F F F
169592024-02-27T21:50:35.967Z 14 280-299 157 157 157 283 283 283 F F F
169602024-02-27T21:50:35.967Z 15 300-319 153 153 153 274 274 274 F F F
169612024-02-27T21:50:35.967Z 16 320-339 156 156 156 278 278 278 F F F
169622024-02-27T21:50:35.967Z 17 340-359 157 157 157 283 283 283 F F F
169632024-02-27T21:50:35.967Z 18 360-379 156 156 156 278 278 278 F F F
169642024-02-27T21:50:35.967Z 19 380-399 156 156 156 282 282 282 F F F
169652024-02-27T21:50:35.967Z 20 400-419 155 155 155 277 277 277 F F F
169662024-02-27T21:50:35.967Z 21 420-439 156 156 156 278 278 278 F F F
169672024-02-27T21:50:35.967Z 22 440-459 148 148 148 269 269 269 F F F
169682024-02-27T21:50:35.967Z 23 460-479 141 141 141 258 258 258 F F F
169692024-02-27T21:50:35.967Z 24 480-499 148 148 148 269 269 269 F F F
169702024-02-27T21:50:35.967Z 25 500-519 148 148 148 269 269 269 F F F
169712024-02-27T21:50:35.967Z 26 520-539 156 156 156 282 282 282 F F F
169722024-02-27T21:50:35.967Z 27 540-559 156 156 156 281 281 281 F F F
169732024-02-27T21:50:35.967Z 28 560-579 156 156 156 281 281 281 F F F
169742024-02-27T21:50:35.967Z 29 580-599 149 149 149 271 271 271 F F F
169752024-02-27T21:50:35.967ZMax gen: 157, Max flush: 283
169762024-02-27T21:50:35.967Z
169772024-02-27T21:50:35.967ZBegin loop 79
169782024-02-27T21:50:35.967ZDownstairs to restart: 2
169792024-02-27T21:50:35.987Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349056233Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782}
169802024-02-27T21:50:35.988Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349311183Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782}
169812024-02-27T21:50:35.988Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349329971Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782}
169822024-02-27T21:50:35.988Z{"msg":"Crucible 49a43d68-b1a2-463e-833e-929035fe132a has session id: 259ce1db-3937-4c37-aae4-d53471bfbe02","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349343909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169832024-02-27T21:50:36.006Z{"msg":"Upstairs opts: Upstairs UUID: 49a43d68-b1a2-463e-833e-929035fe132a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349378525Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169842024-02-27T21:50:36.006Z{"msg":"Crucible runtime is spawned
169852024-02-27T21:50:36.006ZCrucible stats registered with UUID: 49a43d68-b1a2-463e-833e-929035fe132a","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349416471Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169862024-02-27T21:50:36.006Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349473744Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782}
169872024-02-27T21:50:36.006Z{"msg":"49a43d68-b1a2-463e-833e-929035fe132a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.34959325Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169882024-02-27T21:50:36.006Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349621357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169892024-02-27T21:50:36.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349661742Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169902024-02-27T21:50:36.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349685709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169912024-02-27T21:50:36.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349711506Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169922024-02-27T21:50:36.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349736353Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169932024-02-27T21:50:36.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349774599Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169942024-02-27T21:50:36.007Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349800506Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169952024-02-27T21:50:36.008Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349874247Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169962024-02-27T21:50:36.008Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349919682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169972024-02-27T21:50:36.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349956747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169982024-02-27T21:50:36.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.349992143Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
169992024-02-27T21:50:36.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.35001916Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170002024-02-27T21:50:36.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.350741815Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170012024-02-27T21:50:36.008Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.350772071Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170022024-02-27T21:50:36.009Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.350787919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170032024-02-27T21:50:36.009Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.350829805Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170042024-02-27T21:50:36.009Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.350887428Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170052024-02-27T21:50:36.009Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.350922443Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170062024-02-27T21:50:36.009Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.351345974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170072024-02-27T21:50:36.009Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.351445642Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170082024-02-27T21:50:36.010Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.351491456Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170092024-02-27T21:50:36.010Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.351997577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170102024-02-27T21:50:36.010Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352034532Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170112024-02-27T21:50:36.010Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352050581Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170122024-02-27T21:50:36.010Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352072138Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170132024-02-27T21:50:36.011Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352086536Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170142024-02-27T21:50:36.011Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352106864Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170152024-02-27T21:50:36.011Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352120242Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170162024-02-27T21:50:36.011Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.35214058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170172024-02-27T21:50:36.011Z{"msg":"[0]R flush_numbers[0..12]: [273, 281, 272, 283, 272, 283, 271, 279, 276, 272, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352155038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170182024-02-27T21:50:36.012Z{"msg":"[0]R generation[0..12]: [152, 156, 150, 157, 150, 157, 149, 156, 154, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352176186Z","hostname":"Wait for a query_work_queue command to finish before sending IO
170192024-02-27T21:50:36.012Zip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170202024-02-27T21:50:36.012Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352193174Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170212024-02-27T21:50:36.012Z{"msg":"[1]R flush_numbers[0..12]: [273, 281, 272, 283, 272, 283, 271, 279, 276, 272, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352207612Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170222024-02-27T21:50:36.012Z{"msg":"[1]R generation[0..12]: [152, 156, 150, 157, 150, 157, 149, 156, 154, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.35222825Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170232024-02-27T21:50:36.012Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352242598Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170242024-02-27T21:50:36.013Z{"msg":"[2]R flush_numbers[0..12]: [273, 281, 272, 283, 272, 283, 271, 279, 276, 272, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352257516Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170252024-02-27T21:50:36.013Z{"msg":"[2]R generation[0..12]: [152, 156, 150, 157, 150, 157, 149, 156, 154, 150, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352281494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170262024-02-27T21:50:36.013Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.3523113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170272024-02-27T21:50:36.013Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352327108Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170282024-02-27T21:50:36.014Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352353335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170292024-02-27T21:50:36.014Z{"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352372103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170302024-02-27T21:50:36.014Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352385661Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"":"downstairs","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170312024-02-27T21:50:36.014Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352407828Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170322024-02-27T21:50:36.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352420737Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"0","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170332024-02-27T21:50:36.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352446414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"1","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170342024-02-27T21:50:36.015Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352465982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"client":"2","":"downstairs","session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170352024-02-27T21:50:36.015Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352488069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170362024-02-27T21:50:36.015Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
170372024-02-27T21:50:36.015Z,"time":"2024-02-27T21:50:34.352500968Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170382024-02-27T21:50:36.015Z{"msg":"49a43d68-b1a2-463e-833e-929035fe132a is now active with session: 259ce1db-3937-4c37-aae4-d53471bfbe02","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352521505Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170392024-02-27T21:50:36.016Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352533514Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782,"session_id":"259ce1db-3937-4c37-aae4-d53471bfbe02"}
170402024-02-27T21:50:36.016Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:34.352554771Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782}
170412024-02-27T21:50:36.016ZLoading write count information from file "/tmp/test_repair_verify.data"
170422024-02-27T21:50:36.016ZRun Repair workload
170432024-02-27T21:50:36.016Z01/30 Flush
170442024-02-27T21:50:36.016Z02/30 Write block 111 len 20480 data: 16 14 13 10 10
170452024-02-27T21:50:36.017Z03/30 Read block 237 len 40960
170462024-02-27T21:50:36.026Z04/30 Write block 105 len 40960 data: 13 16 18 17 15 14 17 15 14 11
170472024-02-27T21:50:36.026Z05/30 Flush
170482024-02-27T21:50:36.026Z06/30 Write block 399 len 40960 data: 12 15 13 11 11 11 9 9 7 10
170492024-02-27T21:50:36.026Z07/30 Read block 252 len 36864
170502024-02-27T21:50:36.027Z08/30 Read block 356 len 12288
170512024-02-27T21:50:36.027Z09/30 Write block 93 len 20480 data: 12 11 12 10 9
170522024-02-27T21:50:36.027Z10/30 Read block 307 len 32768
170532024-02-27T21:50:36.027Z11/30 Write block 334 len 40960 data: 15 18 14 14 16 15 18 18 15 14
170542024-02-27T21:50:36.027Z12/30 Read block 472 len 8192
170552024-02-27T21:50:36.027Z13/30 Flush
170562024-02-27T21:50:36.028Z14/30 Write block 279 len 32768 data: 8 7 6 7 7 7 5 7
170572024-02-27T21:50:36.028Z15/30 Write block 128 len 24576 data: 13 11 11 9 6 7
170582024-02-27T21:50:36.028Z16/30 Write block 526 len 32768 data: 11 11 10 8 7 7 6 7
170592024-02-27T21:50:36.028Z17/30 Read block 501 len 28672
170602024-02-27T21:50:36.028Z18/30 Read block 345 len 8192
170612024-02-27T21:50:36.028Z19/30 Flush
170622024-02-27T21:50:36.029Z20/30 Write block 527 len 4096 data: 12
170632024-02-27T21:50:36.029Z21/30 Write block 352 len 28672 data: 12 11 10 16 16 10 10
170642024-02-27T21:50:36.029Z22/30 Write block 97 len 12288 data: 10 9 13
170652024-02-27T21:50:36.029Z23/30 Write block 193 len 20480 data: 13 12 11 9 10
170662024-02-27T21:50:36.029Z24/30 Read block 364 len 12288
170672024-02-27T21:50:36.029Z25/30 Read block 123 len 32768
170682024-02-27T21:50:36.029Z26/30 Read block 449 len 40960
170692024-02-27T21:50:36.030Z27/30 Write block 507 len 20480 data: 9 6 11 13 19
170702024-02-27T21:50:36.030Z28/30 Read block 509 len 32768
170712024-02-27T21:50:36.030Z29/30 Read block 456 len 24576
170722024-02-27T21:50:36.038Z30/30 Read block 533 len 24576
170732024-02-27T21:50:36.038Z----------------------------------------------------------------
170742024-02-27T21:50:36.038Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:29
170752024-02-27T21:50:36.038ZStates: Active Active Active
170762024-02-27T21:50:36.038ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170772024-02-27T21:50:36.038Z 2 Acked 1001 Write 5 Done Done Done false
170782024-02-27T21:50:36.038Z 3 Acked 1002 Read 10 Done Done Done false
170792024-02-27T21:50:36.039Z 4 Acked 1003 Write 10 Done Done Sent false
170802024-02-27T21:50:36.039Z 5 Acked 1004 Flush 0 Done Done Sent false
170812024-02-27T21:50:36.039Z 6 Acked 1005 Write 10 Done Done Sent false
170822024-02-27T21:50:36.039Z 7 Acked 1006 Read 9 Done Done Sent false
170832024-02-27T21:50:36.039Z 8 Acked 1007 Read 3 Done Done Sent false
170842024-02-27T21:50:36.039Z 9 Acked 1008 Write 5 Done Done Sent false
170852024-02-27T21:50:36.039Z 10 Acked 1009 Read 8 Done Done Sent false
170862024-02-27T21:50:36.039Z 11 Acked 1010 Write 10 Done Done Sent false
170872024-02-27T21:50:36.039Z 12 Acked 1011 Read 2 Done Done Sent false
170882024-02-27T21:50:36.039Z 13 Acked 1012 Flush 0 Done Done Sent false
170892024-02-27T21:50:36.039Z 14 Acked 1013 Write 8 Done Done Sent false
170902024-02-27T21:50:36.039Z 15 Acked 1014 Write 6 Done Done Sent false
170912024-02-27T21:50:36.039Z 16 Acked 1015 Write 8 Done Done Sent false
170922024-02-27T21:50:36.040Z 17 Acked 1016 Read 7 Done Done Sent false
170932024-02-27T21:50:36.040Z 18 Acked 1017 Read 2 Done Done Sent false
170942024-02-27T21:50:36.040Z 19 Acked 1018 Flush 0 Done Done Sent false
170952024-02-27T21:50:36.040Z 20 Acked 1019 Write 1 Done Done Sent false
170962024-02-27T21:50:36.040Z 21 Acked 1020 Write 7 Done Done Sent false
170972024-02-27T21:50:36.040Z 22 Acked 1021 Write 3 Done Done Sent false
170982024-02-27T21:50:36.040Z 23 Acked 1022 Write 5 Done Done Sent false
170992024-02-27T21:50:36.040Z 24 Acked 1023 Read 3 Done Done Sent false
171002024-02-27T21:50:36.040Z 25 Acked 1024 Read 8 Done Done Sent false
171012024-02-27T21:50:36.040Z 26 Acked 1025 Read 10 Done Done Sent false
171022024-02-27T21:50:36.040Z 27 Acked 1026 Write 5 Done Done Sent false
171032024-02-27T21:50:36.040Z 28 Acked 1027 Read 8 Done Done Sent false
171042024-02-27T21:50:36.040Z 29 Acked 1028 Read 6 Done Done Sent false
171052024-02-27T21:50:36.040Z 30 Acked 1029 Read 6 Done Done Sent false
171062024-02-27T21:50:36.041Z STATES DS:0 DS:1 DS:2 TOTAL
171072024-02-27T21:50:36.041Z New 0 0 0 0
171082024-02-27T21:50:36.041Z Sent 0 0 27 27
171092024-02-27T21:50:36.041Z Done 29 29 2 60
171102024-02-27T21:50:36.041Z Skipped 0 0 0 0
171112024-02-27T21:50:36.041Z Error 0 0 0 0
171122024-02-27T21:50:36.041ZLast Flush: 1018 1018 1000
171132024-02-27T21:50:36.041ZDownstairs last five completed: 1000
171142024-02-27T21:50:36.041ZUpstairs last five completed: 30 29 28 27 26
171152024-02-27T21:50:36.041Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:34.388261355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1782}
171162024-02-27T21:50:36.041ZWrote out file "/tmp/test_repair_verify.data"
171172024-02-27T21:50:36.041Z
171182024-02-27T21:50:36.041ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171192024-02-27T21:50:36.048ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171202024-02-27T21:50:36.049Z 0 000-019 152 152 152 273 273 273 F F F
171212024-02-27T21:50:36.049Z 1 020-039 156 156 156 281 281 281 F F F
171222024-02-27T21:50:36.049Z 2 040-059 150 150 150 272 272 272 F F F
171232024-02-27T21:50:36.049Z 3 060-079 157 157 157 283 283 283 F F F
171242024-02-27T21:50:36.054Z 4 080-099 158 158 150 286 286 272 T T F <---
171252024-02-27T21:50:36.054Z 5 100-119 158 158 157 285 285 283 F F T <---
171262024-02-27T21:50:36.054Z 6 120-139 158 158 149 287 287 271 F F F <---
171272024-02-27T21:50:36.054Z 7 140-159 156 156 156 279 279 279 F F F
171282024-02-27T21:50:36.054Z 8 160-179 154 154 154 276 276 276 F F F
171292024-02-27T21:50:36.054Z 9 180-199 150 150 150 272 272 272 T T F <---
171302024-02-27T21:50:36.054Z 10 200-219 156 156 156 279 279 279 F F F
171312024-02-27T21:50:36.054Z 11 220-239 156 156 156 281 281 281 F F F
171322024-02-27T21:50:36.054Z 12 240-259 155 155 155 277 277 277 F F F
171332024-02-27T21:50:36.054Z 13 260-279 158 158 157 287 287 283 F F F <---
171342024-02-27T21:50:36.054Z 14 280-299 158 158 157 287 287 283 F F F <---
171352024-02-27T21:50:36.054Z 15 300-319 153 153 153 274 274 274 F F F
171362024-02-27T21:50:36.054Z 16 320-339 158 158 156 286 286 278 F F F <---
171372024-02-27T21:50:36.054Z 17 340-359 158 158 157 286 286 283 T T F <---
171382024-02-27T21:50:36.054Z 18 360-379 156 156 156 278 278 278 F F F
171392024-02-27T21:50:36.054Z 19 380-399 158 158 156 286 286 282 F F F <---
171402024-02-27T21:50:36.054Z 20 400-419 158 158 155 286 286 277 F F F <---
171412024-02-27T21:50:36.054Z 21 420-439 156 156 156 278 278 278 F F F
171422024-02-27T21:50:36.054Z 22 440-459 148 148 148 269 269 269 F F F
171432024-02-27T21:50:36.054Z 23 460-479 141 141 141 258 258 258 F F F
171442024-02-27T21:50:36.054Z 24 480-499 148 148 148 269 269 269 F F F
171452024-02-27T21:50:36.055Z 25 500-519 148 148 148 269 269 269 T T F <---
171462024-02-27T21:50:36.055Z 26 520-539 158 158 156 287 287 282 T T F <---
171472024-02-27T21:50:36.055Z 27 540-559 156 156 156 281 281 281 F F F
171482024-02-27T21:50:36.055Z 28 560-579 156 156 156 281 281 281 F F F
171492024-02-27T21:50:36.055Z 29 580-599 149 149 149 271 271 271 F F F
171502024-02-27T21:50:36.055ZMax gen: 158, Max flush: 287
171512024-02-27T21:50:36.055ZError: Difference in extent metadata found!
171522024-02-27T21:50:36.055ZA Difference in extent metadata is expected here
171532024-02-27T21:50:36.055ZOn loop 79
171542024-02-27T21:50:36.055Z
171552024-02-27T21:50:36.061ZVerifying data now
171562024-02-27T21:50:40.653ZNo change after verify
171572024-02-27T21:50:40.653ZLoop: 79 Downstairs dump after verify (and repair):
171582024-02-27T21:50:40.667ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171592024-02-27T21:50:40.667Z 0 000-019 152 152 152 273 273 273 F F F
171602024-02-27T21:50:40.667Z 1 020-039 156 156 156 281 281 281 F F F
171612024-02-27T21:50:40.667Z 2 040-059 150 150 150 272 272 272 F F F
171622024-02-27T21:50:40.667Z 3 060-079 157 157 157 283 283 283 F F F
171632024-02-27T21:50:40.668Z 4 080-099 159 159 159 288 288 288 F F F
171642024-02-27T21:50:40.668Z 5 100-119 158 158 158 285 285 285 F F F
171652024-02-27T21:50:40.668Z 6 120-139 158 158 158 287 287 287 F F F
171662024-02-27T21:50:40.668Z 7 140-159 156 156 156 279 279 279 F F F
171672024-02-27T21:50:40.668Z 8 160-179 154 154 154 276 276 276 F F F
171682024-02-27T21:50:40.668Z 9 180-199 159 159 159 288 288 288 F F F
171692024-02-27T21:50:40.668Z 10 200-219 156 156 156 279 279 279 F F F
171702024-02-27T21:50:40.668Z 11 220-239 156 156 156 281 281 281 F F F
171712024-02-27T21:50:40.668Z 12 240-259 155 155 155 277 277 277 F F F
171722024-02-27T21:50:40.668Z 13 260-279 158 158 158 287 287 287 F F F
171732024-02-27T21:50:40.668Z 14 280-299 158 158 158 287 287 287 F F F
171742024-02-27T21:50:40.668Z 15 300-319 153 153 153 274 274 274 F F F
171752024-02-27T21:50:40.672Z 16 320-339 158 158 158 286 286 286 F F F
171762024-02-27T21:50:40.673Z 17 340-359 159 159 159 288 288 288 F F F
171772024-02-27T21:50:40.673Z 18 360-379 156 156 156 278 278 278 F F F
171782024-02-27T21:50:40.673Z 19 380-399 158 158 158 286 286 286 F F F
171792024-02-27T21:50:40.673Z 20 400-419 158 158 158 286 286 286 F F F
171802024-02-27T21:50:40.673Z 21 420-439 156 156 156 278 278 278 F F F
171812024-02-27T21:50:40.673Z 22 440-459 148 148 148 269 269 269 F F F
171822024-02-27T21:50:40.673Z 23 460-479 141 141 141 258 258 258 F F F
171832024-02-27T21:50:40.673Z 24 480-499 148 148 148 269 269 269 F F F
171842024-02-27T21:50:40.673Z 25 500-519 159 159 159 288 288 288 F F F
171852024-02-27T21:50:40.673Z 26 520-539 159 159 159 288 288 288 F F F
171862024-02-27T21:50:40.673Z 27 540-559 156 156 156 281 281 281 F F F
171872024-02-27T21:50:40.673Z 28 560-579 156 156 156 281 281 281 F F F
171882024-02-27T21:50:40.674Z 29 580-599 149 149 149 271 271 271 F F F
171892024-02-27T21:50:40.674ZMax gen: 159, Max flush: 288
171902024-02-27T21:50:40.674Z
171912024-02-27T21:50:40.674ZBegin loop 80
171922024-02-27T21:50:40.674ZDownstairs to restart: 0
171932024-02-27T21:50:40.693Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.05558883Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790}
171942024-02-27T21:50:40.693Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.05584632Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790}
171952024-02-27T21:50:40.694Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.055865718Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790}
171962024-02-27T21:50:40.694Z{"msg":"Crucible b652d40e-9fa2-4617-aa8b-8bf1dfb9fedb has session id: 79f1d18c-4d76-4db5-8b04-a51d27132f0e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.055886466Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
171972024-02-27T21:50:40.694Z{"msg":"Upstairs opts: Upstairs UUID: b652d40e-9fa2-4617-aa8b-8bf1dfb9fedb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.05593434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
171982024-02-27T21:50:40.701Z{"msg":"Crucible stats registered with UUID: b652d40e-9fa2-4617-aa8b-8bf1dfb9fedb","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.055962346Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
171992024-02-27T21:50:40.701ZCrucible runtime is spawned
172002024-02-27T21:50:40.702Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056117428Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790}
172012024-02-27T21:50:40.702Z{"msg":"b652d40e-9fa2-4617-aa8b-8bf1dfb9fedb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056253802Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172022024-02-27T21:50:40.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056290758Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172032024-02-27T21:50:40.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056336862Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172042024-02-27T21:50:40.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056379108Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172052024-02-27T21:50:40.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056417963Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172062024-02-27T21:50:40.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056482055Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172072024-02-27T21:50:40.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056520051Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172082024-02-27T21:50:40.702Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056547178Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172092024-02-27T21:50:40.702Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056581104Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172102024-02-27T21:50:40.702Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056618119Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172112024-02-27T21:50:40.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056658764Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172122024-02-27T21:50:40.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.05669344Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172132024-02-27T21:50:40.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.056733846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172142024-02-27T21:50:40.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.057231687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172152024-02-27T21:50:40.702Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.057275442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172162024-02-27T21:50:40.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.057320366Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172172024-02-27T21:50:40.703Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.057351653Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172182024-02-27T21:50:40.703Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.057870372Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172192024-02-27T21:50:40.703Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.058026853Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172202024-02-27T21:50:40.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.058262246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172212024-02-27T21:50:40.703Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.058334597Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172222024-02-27T21:50:40.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.058458832Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172232024-02-27T21:50:40.703Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.058508317Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172242024-02-27T21:50:40.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.058525835Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172252024-02-27T21:50:40.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.058549342Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172262024-02-27T21:50:40.703Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.05856397Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172272024-02-27T21:50:40.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.058586767Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172282024-02-27T21:50:40.703Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.058853346Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172292024-02-27T21:50:40.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059275056Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172302024-02-27T21:50:40.703Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059323061Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172312024-02-27T21:50:40.703Z{"msg":"[0]R flush_numbers[0..12]: [273, 281, 272, 283, 288, 285, 287, 279, 276, 288, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059346148Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172322024-02-27T21:50:40.703Z{"msg":"[0]R generation[0..12]: [152, 156, 150, 157, 159, 158, 158, 156, 154, 159, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059369775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172332024-02-27T21:50:40.703Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059386553Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172342024-02-27T21:50:40.703Z{"msg":"[1]R flush_numbers[0..12]: [273, 281, 272, 283, 288, 285, 287, 279, 276, 288, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059409171Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172352024-02-27T21:50:40.703Z{"msg":"[1]R generation[0..12]: [152, 156, 150, 157, 159, 158, 158, 156, 154, 159, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059423959Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172362024-02-27T21:50:40.704Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059445486Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172372024-02-27T21:50:40.704ZWait for a query_work_queue command to finish before sending IO
172382024-02-27T21:50:40.704Z{"msg":"[2]R flush_numbers[0..12]: [273, 281, 272, 283, 288, 285, 287, 279, 276, 288, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059463824Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172392024-02-27T21:50:40.704Z{"msg":"[2]R generation[0..12]: [152, 156, 150, 157, 159, 158, 158, 156, 154, 159, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059479422Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172402024-02-27T21:50:40.704Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.05949413Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172412024-02-27T21:50:40.704Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059515648Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172422024-02-27T21:50:40.704Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059536316Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172432024-02-27T21:50:40.704Z{"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059550234Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172442024-02-27T21:50:40.704Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059571771Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"":"downstairs","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172452024-02-27T21:50:40.704Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059586779Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172462024-02-27T21:50:40.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059610967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"0","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172472024-02-27T21:50:40.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059625085Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"1","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172482024-02-27T21:50:40.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059642113Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"client":"2","":"downstairs","session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172492024-02-27T21:50:40.704Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.05967004Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172502024-02-27T21:50:40.704Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059684318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172512024-02-27T21:50:40.704Z{"msg":"b652d40e-9fa2-4617-aa8b-8bf1dfb9fedb is now active with session: 79f1d18c-4d76-4db5-8b04-a51d27132f0e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059698326Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172522024-02-27T21:50:40.704Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059718104Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790,"session_id":"79f1d18c-4d76-4db5-8b04-a51d27132f0e"}
172532024-02-27T21:50:40.704Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:39.059730843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790}
172542024-02-27T21:50:40.704ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172552024-02-27T21:50:40.704ZLoading write count information from file "/tmp/test_repair_verify.data"
172562024-02-27T21:50:40.704ZRun Repair workload
172572024-02-27T21:50:40.704Z01/30 Write block 515 len 28672 data: 11 12 15 12 11 14 11
172582024-02-27T21:50:40.708Z02/30 Write block 415 len 8192 data: 13 14
172592024-02-27T21:50:40.708Z03/30 Write block 358 len 24576 data: 11 11 10 7 7 11
172602024-02-27T21:50:40.708Z04/30 Read block 358 len 12288
172612024-02-27T21:50:40.708Z05/30 Read block 388 len 32768
172622024-02-27T21:50:40.708Z06/30 Read block 281 len 4096
172632024-02-27T21:50:40.708Z07/30 Read block 405 len 8192
172642024-02-27T21:50:40.708Z08/30 Write block 142 len 24576 data: 12 11 10 9 10 8
172652024-02-27T21:50:40.708Z09/30 Read block 99 len 12288
172662024-02-27T21:50:40.709Z10/30 Read block 446 len 12288
172672024-02-27T21:50:40.709Z11/30 Write block 469 len 28672 data: 12 8 9 8 9 8 10
172682024-02-27T21:50:40.709Z12/30 Read block 248 len 36864
172692024-02-27T21:50:40.709Z13/30 Read block 20 len 32768
172702024-02-27T21:50:40.722Z14/30 Write block 96 len 20480 data: 11 11 10 14 14
172712024-02-27T21:50:40.722Z15/30 Write block 470 len 16384 data: 9 10 9 10
172722024-02-27T21:50:40.722Z16/30 Read block 548 len 24576
172732024-02-27T21:50:40.722Z17/30 Flush
172742024-02-27T21:50:40.726Z18/30 Write block 70 len 28672 data: 11 11 14 11 12 13 15
172752024-02-27T21:50:40.726Z19/30 Read block 570 len 4096
172762024-02-27T21:50:40.726Z20/30 Write block 580 len 28672 data: 10 9 11 12 13 13 13
172772024-02-27T21:50:40.726Z21/30 Flush
172782024-02-27T21:50:40.726Z22/30 Flush
172792024-02-27T21:50:40.726Z23/30 Write block 114 len 16384 data: 12 11 10 12
172802024-02-27T21:50:40.726Z24/30 Read block 465 len 32768
172812024-02-27T21:50:40.726Z25/30 Write block 299 len 20480 data: 11 13 14 15 13
172822024-02-27T21:50:40.726Z26/30 Write block 546 len 36864 data: 10 10 16 18 17 19 16 19 19
172832024-02-27T21:50:40.726Z27/30 Read block 498 len 8192
172842024-02-27T21:50:40.726Z28/30 Read block 408 len 12288
172852024-02-27T21:50:40.731Z29/30 Write block 11 len 36864 data: 16 17 15 15 16 17 16 16 15
172862024-02-27T21:50:40.731Z30/30 Read block 204 len 28672
172872024-02-27T21:50:40.731Z----------------------------------------------------------------
172882024-02-27T21:50:40.731Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
172892024-02-27T21:50:40.731ZStates: Active Active Active
172902024-02-27T21:50:40.731ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172912024-02-27T21:50:40.731Z 1 Acked 1000 Write 7 Done Done Done false
172922024-02-27T21:50:40.731Z 2 Acked 1001 Write 2 Done Done Done false
172932024-02-27T21:50:40.731Z 3 Acked 1002 Write 6 Sent Done Done false
172942024-02-27T21:50:40.731Z 4 Acked 1003 Read 3 Sent Done Done false
172952024-02-27T21:50:40.731Z 5 Acked 1004 Read 8 Sent Done Done false
172962024-02-27T21:50:40.731Z 6 Acked 1005 Read 1 Sent Done Done false
172972024-02-27T21:50:40.732Z 7 Acked 1006 Read 2 Sent Done Done false
172982024-02-27T21:50:40.732Z 8 Acked 1007 Write 6 Sent Done Done false
172992024-02-27T21:50:40.732Z 9 Acked 1008 Read 3 Sent Done Done false
173002024-02-27T21:50:40.732Z 10 Acked 1009 Read 3 Sent Done Done false
173012024-02-27T21:50:40.732Z 11 Acked 1010 Write 7 Sent Done Done false
173022024-02-27T21:50:40.732Z 12 Acked 1011 Read 9 Sent Done Done false
173032024-02-27T21:50:40.732Z 13 Acked 1012 Read 8 Sent Done Done false
173042024-02-27T21:50:40.732Z 14 Acked 1013 Write 5 Sent Done Done false
173052024-02-27T21:50:40.732Z 15 Acked 1014 Write 4 Sent Done Done false
173062024-02-27T21:50:40.732Z 16 Acked 1015 Read 6 Sent Done Done false
173072024-02-27T21:50:40.732Z 17 Acked 1016 Flush 0 Sent Done Done false
173082024-02-27T21:50:40.732Z 18 Acked 1017 Write 7 Sent Done Done false
173092024-02-27T21:50:40.732Z 19 Acked 1018 Read 1 Sent Done Done false
173102024-02-27T21:50:40.732Z 20 Acked 1019 Write 7 Sent Done Done false
173112024-02-27T21:50:40.732Z 21 Acked 1020 Flush 0 Sent Done Done false
173122024-02-27T21:50:40.733Z 22 Acked 1021 Flush 0 Sent Done Done false
173132024-02-27T21:50:40.733Z 23 Acked 1022 Write 4 Sent Done Done false
173142024-02-27T21:50:40.733Z 24 Acked 1023 Read 8 Sent Done Done false
173152024-02-27T21:50:40.733Z 25 Acked 1024 Write 5 Sent Done Done false
173162024-02-27T21:50:40.733Z 26 Acked 1025 Write 9 Sent Done Done false
173172024-02-27T21:50:40.733Z 27 Acked 1026 Read 2 Sent Done Done false
173182024-02-27T21:50:40.733Z 28 Acked 1027 Read 3 Sent Done Done false
173192024-02-27T21:50:40.733Z 29 Acked 1028 Write 9 Sent Done Done false
173202024-02-27T21:50:40.733Z 30 Acked 1029 Read 7 Sent Done Done false
173212024-02-27T21:50:40.733Z STATES DS:0 DS:1 DS:2 TOTAL
173222024-02-27T21:50:40.733Z New 0 0 0 0
173232024-02-27T21:50:40.733Z Sent 28 0 0 28
173242024-02-27T21:50:40.733Z Done 2 30 30 62
173252024-02-27T21:50:40.733Z Skipped 0 0 0 0
173262024-02-27T21:50:40.734Z Error 0 0 0 0
173272024-02-27T21:50:40.734ZLast Flush: 0 1021 1021
173282024-02-27T21:50:40.734ZDownstairs last five completed:
173292024-02-27T21:50:40.734ZUpstairs last five completed: 30 29 28 27 26
173302024-02-27T21:50:40.734Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:39.091791186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1790}
173312024-02-27T21:50:40.734ZWrote out file "/tmp/test_repair_verify.data"
173322024-02-27T21:50:40.739Z
173332024-02-27T21:50:40.739ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
173342024-02-27T21:50:40.766ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173352024-02-27T21:50:40.766Z 0 000-019 152 152 152 273 273 273 F T T <---
173362024-02-27T21:50:40.766Z 1 020-039 156 156 156 281 281 281 F F F
173372024-02-27T21:50:40.766Z 2 040-059 150 150 150 272 272 272 F F F
173382024-02-27T21:50:40.766Z 3 060-079 157 160 160 283 290 290 F F F <---
173392024-02-27T21:50:40.766Z 4 080-099 159 160 160 288 289 289 F F F <---
173402024-02-27T21:50:40.766Z 5 100-119 158 160 160 285 289 289 F T T <---
173412024-02-27T21:50:40.766Z 6 120-139 158 158 158 287 287 287 F F F
173422024-02-27T21:50:40.770Z 7 140-159 156 160 160 279 289 289 F F F <---
173432024-02-27T21:50:40.770Z 8 160-179 154 154 154 276 276 276 F F F
173442024-02-27T21:50:40.770Z 9 180-199 159 159 159 288 288 288 F F F
173452024-02-27T21:50:40.770Z 10 200-219 156 156 156 279 279 279 F F F
173462024-02-27T21:50:40.770Z 11 220-239 156 156 156 281 281 281 F F F
173472024-02-27T21:50:40.770Z 12 240-259 155 155 155 277 277 277 F F F
173482024-02-27T21:50:40.771Z 13 260-279 158 158 158 287 287 287 F F F
173492024-02-27T21:50:40.771Z 14 280-299 158 158 158 287 287 287 F T T <---
173502024-02-27T21:50:40.771Z 15 300-319 153 153 153 274 274 274 F T T <---
173512024-02-27T21:50:40.771Z 16 320-339 158 158 158 286 286 286 F F F
173522024-02-27T21:50:40.771Z 17 340-359 159 160 160 288 289 289 F F F <---
173532024-02-27T21:50:40.771Z 18 360-379 156 160 160 278 289 289 F F F <---
173542024-02-27T21:50:40.771Z 19 380-399 158 158 158 286 286 286 F F F
173552024-02-27T21:50:40.771Z 20 400-419 158 160 160 286 289 289 T F F <---
173562024-02-27T21:50:40.771Z 21 420-439 156 156 156 278 278 278 F F F
173572024-02-27T21:50:40.771Z 22 440-459 148 148 148 269 269 269 F F F
173582024-02-27T21:50:40.771Z 23 460-479 141 160 160 258 289 289 F F F <---
173592024-02-27T21:50:40.771Z 24 480-499 148 148 148 269 269 269 F F F
173602024-02-27T21:50:40.771Z 25 500-519 159 160 160 288 289 289 T F F <---
173612024-02-27T21:50:40.772Z 26 520-539 159 160 160 288 289 289 T F F <---
173622024-02-27T21:50:40.772Z 27 540-559 156 156 156 281 281 281 F T T <---
173632024-02-27T21:50:40.772Z 28 560-579 156 156 156 281 281 281 F F F
173642024-02-27T21:50:40.772Z 29 580-599 149 160 160 271 290 290 F F F <---
173652024-02-27T21:50:40.772ZMax gen: 160, Max flush: 290
173662024-02-27T21:50:40.772ZError: Difference in extent metadata found!
173672024-02-27T21:50:40.772ZA Difference in extent metadata is expected here
173682024-02-27T21:50:40.772ZOn loop 80
173692024-02-27T21:50:40.772Z
173702024-02-27T21:50:40.772ZVerifying data now
173712024-02-27T21:50:46.491ZNo change after verify
173722024-02-27T21:50:46.491ZLoop: 80 Downstairs dump after verify (and repair):
173732024-02-27T21:50:46.498ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173742024-02-27T21:50:46.498Z 0 000-019 161 161 161 291 291 291 F F F
173752024-02-27T21:50:46.499Z 1 020-039 156 156 156 281 281 281 F F F
173762024-02-27T21:50:46.499Z 2 040-059 150 150 150 272 272 272 F F F
173772024-02-27T21:50:46.499Z 3 060-079 160 160 160 290 290 290 F F F
173782024-02-27T21:50:46.499Z 4 080-099 160 160 160 289 289 289 F F F
173792024-02-27T21:50:46.503Z 5 100-119 161 161 161 291 291 291 F F F
173802024-02-27T21:50:46.503Z 6 120-139 158 158 158 287 287 287 F F F
173812024-02-27T21:50:46.503Z 7 140-159 160 160 160 289 289 289 F F F
173822024-02-27T21:50:46.503Z 8 160-179 154 154 154 276 276 276 F F F
173832024-02-27T21:50:46.503Z 9 180-199 159 159 159 288 288 288 F F F
173842024-02-27T21:50:46.503Z 10 200-219 156 156 156 279 279 279 F F F
173852024-02-27T21:50:46.503Z 11 220-239 156 156 156 281 281 281 F F F
173862024-02-27T21:50:46.503Z 12 240-259 155 155 155 277 277 277 F F F
173872024-02-27T21:50:46.503Z 13 260-279 158 158 158 287 287 287 F F F
173882024-02-27T21:50:46.503Z 14 280-299 161 161 161 291 291 291 F F F
173892024-02-27T21:50:46.503Z 15 300-319 161 161 161 291 291 291 F F F
173902024-02-27T21:50:46.503Z 16 320-339 158 158 158 286 286 286 F F F
173912024-02-27T21:50:46.503Z 17 340-359 160 160 160 289 289 289 F F F
173922024-02-27T21:50:46.503Z 18 360-379 160 160 160 289 289 289 F F F
173932024-02-27T21:50:46.503Z 19 380-399 158 158 158 286 286 286 F F F
173942024-02-27T21:50:46.503Z 20 400-419 160 160 160 289 289 289 F F F
173952024-02-27T21:50:46.503Z 21 420-439 156 156 156 278 278 278 F F F
173962024-02-27T21:50:46.503Z 22 440-459 148 148 148 269 269 269 F F F
173972024-02-27T21:50:46.504Z 23 460-479 160 160 160 289 289 289 F F F
173982024-02-27T21:50:46.504Z 24 480-499 148 148 148 269 269 269 F F F
173992024-02-27T21:50:46.504Z 25 500-519 160 160 160 289 289 289 F F F
174002024-02-27T21:50:46.504Z 26 520-539 160 160 160 289 289 289 F F F
174012024-02-27T21:50:46.504Z 27 540-559 161 161 161 291 291 291 F F F
174022024-02-27T21:50:46.504Z 28 560-579 156 156 156 281 281 281 F F F
174032024-02-27T21:50:46.504Z 29 580-599 160 160 160 290 290 290 F F F
174042024-02-27T21:50:46.504ZMax gen: 161, Max flush: 291
174052024-02-27T21:50:46.504Z
174062024-02-27T21:50:46.504ZBegin loop 81
174072024-02-27T21:50:46.504ZDownstairs to restart: 1
174082024-02-27T21:50:46.521Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.883347434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798}
174092024-02-27T21:50:46.521Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.883674516Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798}
174102024-02-27T21:50:46.521Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0Crucible runtime is spawned
174112024-02-27T21:50:46.529Z,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.883733929Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798}
174122024-02-27T21:50:46.529Z{"msg":"Crucible f741a882-80d3-43a2-913e-a8fbbbb5da07 has session id: 4d62818a-6c3a-40ce-a436-d8b517e07496","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.883765425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174132024-02-27T21:50:46.529Z{"msg":"Upstairs opts: Upstairs UUID: f741a882-80d3-43a2-913e-a8fbbbb5da07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.883821538Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174142024-02-27T21:50:46.529Z{"msg":"Crucible stats registered with UUID: f741a882-80d3-43a2-913e-a8fbbbb5da07","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.883866523Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174152024-02-27T21:50:46.529Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.883907958Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798}
174162024-02-27T21:50:46.529Z{"msg":"f741a882-80d3-43a2-913e-a8fbbbb5da07 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.883984649Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174172024-02-27T21:50:46.529Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884035093Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174182024-02-27T21:50:46.529Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884078738Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174192024-02-27T21:50:46.529Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884119793Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174202024-02-27T21:50:46.529Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.88414829Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174212024-02-27T21:50:46.529Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884198164Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174222024-02-27T21:50:46.529Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.88423213Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174232024-02-27T21:50:46.529Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884281564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174242024-02-27T21:50:46.529Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884331788Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174252024-02-27T21:50:46.529Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884358095Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174262024-02-27T21:50:46.529Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884385242Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174272024-02-27T21:50:46.530Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.88457134Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174282024-02-27T21:50:46.530Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884632963Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174292024-02-27T21:50:46.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.884990881Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174302024-02-27T21:50:46.530Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.885030766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174312024-02-27T21:50:46.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.885156441Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174322024-02-27T21:50:46.530Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.885230653Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174332024-02-27T21:50:46.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.885288086Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174342024-02-27T21:50:46.530Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.885320012Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174352024-02-27T21:50:46.530Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.885719245Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174362024-02-27T21:50:46.530Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.885818673Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174372024-02-27T21:50:46.530Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.88609956Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174382024-02-27T21:50:46.530Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886276999Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174392024-02-27T21:50:46.530Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886318324Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174402024-02-27T21:50:46.530Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886336112Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174412024-02-27T21:50:46.530Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.88635663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174422024-02-27T21:50:46.530Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886370748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174432024-02-27T21:50:46.530Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886384426Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174442024-02-27T21:50:46.530Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886578414Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174452024-02-27T21:50:46.530Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.88661429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174462024-02-27T21:50:46.530Z{"msg":"[0]R flush_numbers[0..12]: [291, 281, 272, 290, 289, 291, 287, 289, 276, 288, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886641526Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174472024-02-27T21:50:46.530Z{"msg":"[0]R generation[0..12]: [161, 156, 150, 160, 160, 161, 158, 160, 154, 159, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886668133Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174482024-02-27T21:50:46.530Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886684151Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174492024-02-27T21:50:46.530Z{"msg":"[1]R flush_numbers[0..12]: [291, 281, 272, 290, 289, 291, 287, 289, 276, 288, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886705329Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174502024-02-27T21:50:46.531Z{"msg":"[1]R generation[0..12]: [161, 156, 150, 160, 160, 161, 158, 160, 154, 159, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886720547Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174512024-02-27T21:50:46.531ZWait for a query_work_queue command to finish before sending IO
174522024-02-27T21:50:46.531Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886747194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174532024-02-27T21:50:46.531Z{"msg":"[2]R flush_numbers[0..12]: [291, 281, 272, 290, 289, 291, 287, 289, 276, 288, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886762502Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174542024-02-27T21:50:46.531Z{"msg":"[2]R generation[0..12]: [161, 156, 150, 160, 160, 161, 158, 160, 154, 159, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.88678355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174552024-02-27T21:50:46.531Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886797888Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174562024-02-27T21:50:46.531Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886817766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174572024-02-27T21:50:46.531Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886831684Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174582024-02-27T21:50:46.531Z{"msg":"Next flush: 292","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886851831Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174592024-02-27T21:50:46.531Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.88686558Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"":"downstairs","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174602024-02-27T21:50:46.531Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886879368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174612024-02-27T21:50:46.531Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886898526Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"0","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174622024-02-27T21:50:46.531Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886912084Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"1","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174632024-02-27T21:50:46.531Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886931622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"client":"2","":"downstairs","session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174642024-02-27T21:50:46.531Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.88694514Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174652024-02-27T21:50:46.531Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886957859Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174662024-02-27T21:50:46.532Z{"msg":"f741a882-80d3-43a2-913e-a8fbbbb5da07 is now active with session: 4d62818a-6c3a-40ce-a436-d8b517e07496","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886977767Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174672024-02-27T21:50:46.532Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.886989985Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798,"session_id":"4d62818a-6c3a-40ce-a436-d8b517e07496"}
174682024-02-27T21:50:46.532Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:44.887024991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798}
174692024-02-27T21:50:46.532ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174702024-02-27T21:50:46.557ZLoading write count information from file "/tmp/test_repair_verify.data"
174712024-02-27T21:50:46.557ZRun Repair workload
174722024-02-27T21:50:46.558Z01/30 Write block 262 len 20480 data: 12 13 15 15 10
174732024-02-27T21:50:46.558Z02/30 Write block 401 len 32768 data: 14 12 12 12 10 10 8 11
174742024-02-27T21:50:46.558Z03/30 Read block 472 len 40960
174752024-02-27T21:50:46.558Z04/30 Write block 163 len 32768 data: 12 10 10 7 8 7 9 10
174762024-02-27T21:50:46.558Z05/30 Write block 21 len 20480 data: 14 10 8 13 11
174772024-02-27T21:50:46.558Z06/30 Flush
174782024-02-27T21:50:46.558Z07/30 Write block 138 len 20480 data: 10 11 11 11 13
174792024-02-27T21:50:46.558Z08/30 Write block 80 len 32768 data: 9 9 8 7 9 9 10 13
174802024-02-27T21:50:46.558Z09/30 Read block 392 len 40960
174812024-02-27T21:50:46.558Z10/30 Read block 490 len 28672
174822024-02-27T21:50:46.558Z11/30 Write block 260 len 12288 data: 13 12 13
174832024-02-27T21:50:46.558Z12/30 Read block 30 len 36864
174842024-02-27T21:50:46.558Z13/30 Flush
174852024-02-27T21:50:46.564Z14/30 Write block 272 len 40960 data: 13 9 10 9 11 10 9 9 8 7
174862024-02-27T21:50:46.564Z15/30 Write block 351 len 20480 data: 16 13 12 11 17
174872024-02-27T21:50:46.564Z16/30 Read block 447 len 32768
174882024-02-27T21:50:46.564Z17/30 Flush
174892024-02-27T21:50:46.564Z18/30 Write block 147 len 36864 data: 9 10 12 12 10 10 9 9 12
174902024-02-27T21:50:46.564Z19/30 Write block 389 len 40960 data: 13 13 15 13 14 11 12 11 14 12
174912024-02-27T21:50:46.565Z20/30 Flush
174922024-02-27T21:50:46.565Z21/30 Flush
174932024-02-27T21:50:46.565Z22/30 Write block 480 len 16384 data: 10 11 13 15
174942024-02-27T21:50:46.565Z23/30 Read block 560 len 4096
174952024-02-27T21:50:46.565Z24/30 Write block 421 len 8192 data: 8 9
174962024-02-27T21:50:46.565Z25/30 Write block 486 len 16384 data: 16 17 15 17
174972024-02-27T21:50:46.565Z26/30 Read block 248 len 28672
174982024-02-27T21:50:46.565Z27/30 Read block 31 len 12288
174992024-02-27T21:50:46.565Z28/30 Write block 310 len 20480 data: 16 15 14 12 13
175002024-02-27T21:50:46.565Z29/30 Write block 51 len 24576 data: 14 15 14 12 8 13
175012024-02-27T21:50:46.565Z30/30 Read block 357 len 16384
175022024-02-27T21:50:46.565Z----------------------------------------------------------------
175032024-02-27T21:50:46.565Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
175042024-02-27T21:50:46.566ZStates: Active Active Active
175052024-02-27T21:50:46.566ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
175062024-02-27T21:50:46.566Z 1 Acked 1000 Write 5 Done Done Done false
175072024-02-27T21:50:46.566Z 2 Acked 1001 Write 8 Done Done Done false
175082024-02-27T21:50:46.566Z 3 Acked 1002 Read 10 Done Done Done false
175092024-02-27T21:50:46.566Z 4 Acked 1003 Write 8 Done Done Done false
175102024-02-27T21:50:46.566Z 5 Acked 1004 Write 5 Done Sent Done false
175112024-02-27T21:50:46.566Z 6 Acked 1005 Flush 0 Done Sent Done false
175122024-02-27T21:50:46.566Z 7 Acked 1006 Write 5 Done Sent Done false
175132024-02-27T21:50:46.566Z 8 Acked 1007 Write 8 Done Sent Done false
175142024-02-27T21:50:46.566Z 9 Acked 1008 Read 10 Done Sent Done false
175152024-02-27T21:50:46.566Z 10 Acked 1009 Read 7 Done Sent Done false
175162024-02-27T21:50:46.566Z 11 Acked 1010 Write 3 Done Sent Done false
175172024-02-27T21:50:46.567Z 12 Acked 1011 Read 9 Done Sent Done false
175182024-02-27T21:50:46.567Z 13 Acked 1012 Flush 0 Done Sent Done false
175192024-02-27T21:50:46.567Z 14 Acked 1013 Write 10 Done Sent Done false
175202024-02-27T21:50:46.567Z 15 Acked 1014 Write 5 Done Sent Done false
175212024-02-27T21:50:46.567Z 16 Acked 1015 Read 8 Done Sent Done false
175222024-02-27T21:50:46.567Z 17 Acked 1016 Flush 0 Done Sent Done false
175232024-02-27T21:50:46.567Z 18 Acked 1017 Write 9 Done Sent Done false
175242024-02-27T21:50:46.567Z 19 Acked 1018 Write 10 Done Sent Done false
175252024-02-27T21:50:46.567Z 20 Acked 1019 Flush 0 Done Sent Done false
175262024-02-27T21:50:46.567Z 21 Acked 1020 Flush 0 Done Sent Done false
175272024-02-27T21:50:46.567Z 22 Acked 1021 Write 4 Done Sent Done false
175282024-02-27T21:50:46.567Z 23 Acked 1022 Read 1 Done Sent Done false
175292024-02-27T21:50:46.567Z 24 Acked 1023 Write 2 Done Sent Done false
175302024-02-27T21:50:46.567Z 25 Acked 1024 Write 4 Done Sent Done false
175312024-02-27T21:50:46.568Z 26 Acked 1025 Read 7 Done Sent Done false
175322024-02-27T21:50:46.568Z 27 Acked 1026 Read 3 Done Sent Done false
175332024-02-27T21:50:46.568Z 28 Acked 1027 Write 5 Done Sent Done false
175342024-02-27T21:50:46.568Z 29 Acked 1028 Write 6 Done Sent Done false
175352024-02-27T21:50:46.568Z 30 Acked 1029 Read 4 Done Sent Sent false
175362024-02-27T21:50:46.568Z STATES DS:0 DS:1 DS:2 TOTAL
175372024-02-27T21:50:46.568Z New 0 0 0 0
175382024-02-27T21:50:46.568Z Sent 0 26 1 27
175392024-02-27T21:50:46.568Z Done 30 4 29 63
175402024-02-27T21:50:46.568Z Skipped 0 0 0 0
175412024-02-27T21:50:46.568Z Error 0 0 0 0
175422024-02-27T21:50:46.568ZLast Flush: 1020 0 1020
175432024-02-27T21:50:46.568ZDownstairs last five completed:
175442024-02-27T21:50:46.568ZUpstairs last five completed: 30 29 28 27 26
175452024-02-27T21:50:46.569Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:44.920431606Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1798}
175462024-02-27T21:50:46.569ZWrote out file "/tmp/test_repair_verify.data"
175472024-02-27T21:50:46.569Z
175482024-02-27T21:50:46.573ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175492024-02-27T21:50:46.584ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175502024-02-27T21:50:46.585Z 0 000-019 161 161 161 291 291 291 F F F
175512024-02-27T21:50:46.585Z 1 020-039 162 156 162 292 281 292 F F F <---
175522024-02-27T21:50:46.585Z 2 040-059 150 150 150 272 272 272 T F T <---
175532024-02-27T21:50:46.585Z 3 060-079 160 160 160 290 290 290 F F F
175542024-02-27T21:50:46.585Z 4 080-099 162 160 162 293 289 293 F F F <---
175552024-02-27T21:50:46.585Z 5 100-119 161 161 161 291 291 291 F F F
175562024-02-27T21:50:46.585Z 6 120-139 162 158 162 293 287 293 F F F <---
175572024-02-27T21:50:46.585Z 7 140-159 162 160 162 295 289 295 F F F <---
175582024-02-27T21:50:46.585Z 8 160-179 162 154 162 292 276 292 F T F <---
175592024-02-27T21:50:46.585Z 9 180-199 159 159 159 288 288 288 F F F
175602024-02-27T21:50:46.585Z 10 200-219 156 156 156 279 279 279 F F F
175612024-02-27T21:50:46.585Z 11 220-239 156 156 156 281 281 281 F F F
175622024-02-27T21:50:46.585Z 12 240-259 155 155 155 277 277 277 F F F
175632024-02-27T21:50:46.585Z 13 260-279 162 158 162 294 287 294 F T F <---
175642024-02-27T21:50:46.585Z 14 280-299 162 161 162 294 291 294 F F F <---
175652024-02-27T21:50:46.585Z 15 300-319 161 161 161 291 291 291 T F T <---
175662024-02-27T21:50:46.585Z 16 320-339 158 158 158 286 286 286 F F F
175672024-02-27T21:50:46.590Z 17 340-359 162 160 162 294 289 294 F F F <---
175682024-02-27T21:50:46.590Z 18 360-379 160 160 160 289 289 289 F F F
175692024-02-27T21:50:46.590Z 19 380-399 162 158 162 295 286 295 F F F <---
175702024-02-27T21:50:46.590Z 20 400-419 162 160 162 292 289 292 F T F <---
175712024-02-27T21:50:46.590Z 21 420-439 156 156 156 278 278 278 T F T <---
175722024-02-27T21:50:46.590Z 22 440-459 148 148 148 269 269 269 F F F
175732024-02-27T21:50:46.590Z 23 460-479 160 160 160 289 289 289 F F F
175742024-02-27T21:50:46.590Z 24 480-499 148 148 148 269 269 269 T F T <---
175752024-02-27T21:50:46.590Z 25 500-519 160 160 160 289 289 289 F F F
175762024-02-27T21:50:46.590Z 26 520-539 160 160 160 289 289 289 F F F
175772024-02-27T21:50:46.590Z 27 540-559 161 161 161 291 291 291 F F F
175782024-02-27T21:50:46.590Z 28 560-579 156 156 156 281 281 281 F F F
175792024-02-27T21:50:46.590Z 29 580-599 160 160 160 290 290 290 F F F
175802024-02-27T21:50:46.590ZMax gen: 162, Max flush: 295
175812024-02-27T21:50:46.590ZError: Difference in extent metadata found!
175822024-02-27T21:50:46.590ZA Difference in extent metadata is expected here
175832024-02-27T21:50:46.590ZOn loop 81
175842024-02-27T21:50:46.590Z
175852024-02-27T21:50:46.594ZVerifying data now
175862024-02-27T21:50:52.286ZNo change after verify
175872024-02-27T21:50:52.286ZLoop: 81 Downstairs dump after verify (and repair):
175882024-02-27T21:50:52.302ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175892024-02-27T21:50:52.302Z 0 000-019 161 161 161 291 291 291 F F F
175902024-02-27T21:50:52.302Z 1 020-039 162 162 162 292 292 292 F F F
175912024-02-27T21:50:52.302Z 2 040-059 163 163 163 296 296 296 F F F
175922024-02-27T21:50:52.302Z 3 060-079 160 160 160 290 290 290 F F F
175932024-02-27T21:50:52.302Z 4 080-099 162 162 162 293 293 293 F F F
175942024-02-27T21:50:52.302Z 5 100-119 161 161 161 291 291 291 F F F
175952024-02-27T21:50:52.302Z 6 120-139 162 162 162 293 293 293 F F F
175962024-02-27T21:50:52.302Z 7 140-159 162 162 162 295 295 295 F F F
175972024-02-27T21:50:52.302Z 8 160-179 162 162 162 292 292 292 F F F
175982024-02-27T21:50:52.302Z 9 180-199 159 159 159 288 288 288 F F F
175992024-02-27T21:50:52.302Z 10 200-219 156 156 156 279 279 279 F F F
176002024-02-27T21:50:52.302Z 11 220-239 156 156 156 281 281 281 F F F
176012024-02-27T21:50:52.302Z 12 240-259 155 155 155 277 277 277 F F F
176022024-02-27T21:50:52.302Z 13 260-279 162 162 162 294 294 294 F F F
176032024-02-27T21:50:52.302Z 14 280-299 162 162 162 294 294 294 F F F
176042024-02-27T21:50:52.303Z 15 300-319 163 163 163 296 296 296 F F F
176052024-02-27T21:50:52.303Z 16 320-339 158 158 158 286 286 286 F F F
176062024-02-27T21:50:52.303Z 17 340-359 162 162 162 294 294 294 F F F
176072024-02-27T21:50:52.303Z 18 360-379 160 160 160 289 289 289 F F F
176082024-02-27T21:50:52.306Z 19 380-399 162 162 162 295 295 295 F F F
176092024-02-27T21:50:52.306Z 20 400-419 162 162 162 292 292 292 F F F
176102024-02-27T21:50:52.306Z 21 420-439 163 163 163 296 296 296 F F F
176112024-02-27T21:50:52.306Z 22 440-459 148 148 148 269 269 269 F F F
176122024-02-27T21:50:52.306Z 23 460-479 160 160 160 289 289 289 F F F
176132024-02-27T21:50:52.306Z 24 480-499 163 163 163 296 296 296 F F F
176142024-02-27T21:50:52.306Z 25 500-519 160 160 160 289 289 289 F F F
176152024-02-27T21:50:52.307Z 26 520-539 160 160 160 289 289 289 F F F
176162024-02-27T21:50:52.307Z 27 540-559 161 161 161 291 291 291 F F F
176172024-02-27T21:50:52.307Z 28 560-579 156 156 156 281 281 281 F F F
176182024-02-27T21:50:52.307Z 29 580-599 160 160 160 290 290 290 F F F
176192024-02-27T21:50:52.307ZMax gen: 163, Max flush: 296
176202024-02-27T21:50:52.307Z
176212024-02-27T21:50:52.307ZBegin loop 82
176222024-02-27T21:50:52.307ZDownstairs to restart: 0
176232024-02-27T21:50:52.327Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.688521159Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806}
176242024-02-27T21:50:52.327Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.688800976Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806}
176252024-02-27T21:50:52.327Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.688818194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806}
176262024-02-27T21:50:52.334Z{"msg":"Crucible 5a4a4e53-1f32-4c43-b312-fd1c96743762 has session id: c84a75a9-accb-4faf-9cb8-d498047400d9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.68885041Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176272024-02-27T21:50:52.334ZCrucible runtime is spawned
176282024-02-27T21:50:52.334Z{"msg":"Upstairs opts: Upstairs UUID: 5a4a4e53-1f32-4c43-b312-fd1c96743762, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.688896735Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176292024-02-27T21:50:52.334Z{"msg":"Crucible stats registered with UUID: 5a4a4e53-1f32-4c43-b312-fd1c96743762","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.688929461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176302024-02-27T21:50:52.334Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.688951128Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806}
176312024-02-27T21:50:52.334Z{"msg":"5a4a4e53-1f32-4c43-b312-fd1c96743762 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689058846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176322024-02-27T21:50:52.334Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689092702Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176332024-02-27T21:50:52.334Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689116269Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176342024-02-27T21:50:52.334Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689147155Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176352024-02-27T21:50:52.334Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689172742Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176362024-02-27T21:50:52.334Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689203259Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176372024-02-27T21:50:52.334Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689228186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176382024-02-27T21:50:52.334Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689249053Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176392024-02-27T21:50:52.334Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689283829Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176402024-02-27T21:50:52.334Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689312856Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176412024-02-27T21:50:52.334Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689371289Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176422024-02-27T21:50:52.335Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689408645Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176432024-02-27T21:50:52.335Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.689475686Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176442024-02-27T21:50:52.335Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.690173234Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176452024-02-27T21:50:52.335Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.690216549Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176462024-02-27T21:50:52.335Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.690240307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176472024-02-27T21:50:52.335Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.69029697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176482024-02-27T21:50:52.335Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.690863163Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176492024-02-27T21:50:52.335Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.690910377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176502024-02-27T21:50:52.335Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.690962762Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176512024-02-27T21:50:52.335Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.690995507Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176522024-02-27T21:50:52.335Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.691443395Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176532024-02-27T21:50:52.335Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.69148009Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176542024-02-27T21:50:52.335Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.691496359Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176552024-02-27T21:50:52.335Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.691509277Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176562024-02-27T21:50:52.335Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.691538434Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176572024-02-27T21:50:52.335Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.691560571Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176582024-02-27T21:50:52.335Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.691669648Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176592024-02-27T21:50:52.335Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692081939Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176602024-02-27T21:50:52.335Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692143462Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176612024-02-27T21:50:52.335Z{"msg":"[0]R flush_numbers[0..12]: [291, 292, 296, 290, 293, 291, 293, 295, 292, 288, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692168909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176622024-02-27T21:50:52.335Z{"msg":"[0]R generation[0..12]: [161, 162, 163, 160, 162, 161, 162, 162, 162, 159, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692199656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176632024-02-27T21:50:52.335Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692219783Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176642024-02-27T21:50:52.335Z{"msg":"[1]R flush_numbers[0..12]: [291, 292, 296, 290, 293, 291, 293, 295, 292, 288, 279, 281]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
176652024-02-27T21:50:52.335Z,"time":"2024-02-27T21:50:50.692255879Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176662024-02-27T21:50:52.336Z{"msg":"[1]R generation[0..12]: [161, 162, 163, 160, 162, 161, 162, 162, 162, 159, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692282936Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176672024-02-27T21:50:52.336Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692314562Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176682024-02-27T21:50:52.336Z{"msg":"[2]R flush_numbers[0..12]: [291, 292, 296, 290, 293, 291, 293, 295, 292, 288, 279, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.69233581Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176692024-02-27T21:50:52.336Z{"msg":"[2]R generation[0..12]: [161, 162, 163, 160, 162, 161, 162, 162, 162, 159, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692365426Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176702024-02-27T21:50:52.336Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692396433Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176712024-02-27T21:50:52.336Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.69241502Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176722024-02-27T21:50:52.336Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692444987Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176732024-02-27T21:50:52.336Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692470504Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176742024-02-27T21:50:52.336Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692494971Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"":"downstairs","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176752024-02-27T21:50:52.336Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692526227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176762024-02-27T21:50:52.336Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692542285Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"0","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176772024-02-27T21:50:52.336Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692573092Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"1","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176782024-02-27T21:50:52.336Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692606328Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"client":"2","":"downstairs","session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176792024-02-27T21:50:52.336Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692638324Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176802024-02-27T21:50:52.336Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176812024-02-27T21:50:52.336Z,"level":30,"time":"2024-02-27T21:50:50.692677919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176822024-02-27T21:50:52.336Z{"msg":"5a4a4e53-1f32-4c43-b312-fd1c96743762 is now active with session: c84a75a9-accb-4faf-9cb8-d498047400d9","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692709546Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176832024-02-27T21:50:52.340Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692723664Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806,"session_id":"c84a75a9-accb-4faf-9cb8-d498047400d9"}
176842024-02-27T21:50:52.340Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:50.692753491Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806}
176852024-02-27T21:50:52.340ZLoading write count information from file "/tmp/test_repair_verify.data"
176862024-02-27T21:50:52.340ZRun Repair workload
176872024-02-27T21:50:52.340Z01/30 Write block 306 len 12288 data: 13 15 14
176882024-02-27T21:50:52.340Z02/30 Read block 536 len 32768
176892024-02-27T21:50:52.340Z03/30 Write block 569 len 40960 data: 10 8 9 11 12 10 9 9 11 13
176902024-02-27T21:50:52.340Z04/30 Write block 38 len 40960 data: 7 9 11 11 11 13 14 16 12 12
176912024-02-27T21:50:52.340Z05/30 Read block 122 len 16384
176922024-02-27T21:50:52.341Z06/30 Read block 282 len 24576
176932024-02-27T21:50:52.341Z07/30 Write block 222 len 16384 data: 13 11 11 10
176942024-02-27T21:50:52.341Z08/30 Flush
176952024-02-27T21:50:52.344Z09/30 Write block 287 len 28672 data: 10 11 14 14 14 13 17
176962024-02-27T21:50:52.344Z10/30 Write block 344 len 20480 data: 15 13 14 14 18
176972024-02-27T21:50:52.344Z11/30 Write block 213 len 28672 data: 12 14 16 16 18 17 15
176982024-02-27T21:50:52.344Z12/30 Write block 194 len 4096 data: 13
176992024-02-27T21:50:52.344Z13/30 Write block 46 len 8192 data: 13 13
177002024-02-27T21:50:52.348Z14/30 Write block 55 len 40960 data: 9 14 13 12 11 9 8 9 10 9
177012024-02-27T21:50:52.348Z15/30 Write block 321 len 28672 data: 16 16 15 13 11 9 11
177022024-02-27T21:50:52.348Z16/30 Read block 404 len 20480
177032024-02-27T21:50:52.348Z17/30 Read block 161 len 4096
177042024-02-27T21:50:52.353Z18/30 Read block 222 len 40960
177052024-02-27T21:50:52.353Z19/30 Read block 493 len 8192
177062024-02-27T21:50:52.353Z20/30 Flush
177072024-02-27T21:50:52.357Z21/30 Write block 339 len 40960 data: 16 19 19 16 15 16 14 15 15 19
177082024-02-27T21:50:52.357Z22/30 Read block 250 len 24576
177092024-02-27T21:50:52.357Z23/30 Write block 575 len 20480 data: 10 10 12 14 10
177102024-02-27T21:50:52.357Z24/30 Read block 136 len 8192
177112024-02-27T21:50:52.361Z25/30 Write block 543 len 36864 data: 11 9 10 11 11 17 19 18 20
177122024-02-27T21:50:52.361Z26/30 Flush
177132024-02-27T21:50:52.365Z27/30 Write block 291 len 16384 data: 15 14 18 17
177142024-02-27T21:50:52.366Z28/30 Read block 150 len 16384
177152024-02-27T21:50:52.366Z29/30 Write block 302 len 12288 data: 16 14 16
177162024-02-27T21:50:52.366Z30/30 Read block 31 len 36864
177172024-02-27T21:50:52.366Z----------------------------------------------------------------
177182024-02-27T21:50:52.366Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
177192024-02-27T21:50:52.366ZStates: Active Active Active
177202024-02-27T21:50:52.366ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177212024-02-27T21:50:52.366Z 1 Acked 1000 Write 3 Sent Done Done false
177222024-02-27T21:50:52.366Z 2 Acked 1001 Read 8 Sent Done Done false
177232024-02-27T21:50:52.366Z 3 Acked 1002 Write 10 Sent Done Done false
177242024-02-27T21:50:52.366Z 4 Acked 1003 Write 10 Sent Done Done false
177252024-02-27T21:50:52.366Z 5 Acked 1004 Read 4 Sent Done Done false
177262024-02-27T21:50:52.367Z 6 Acked 1005 Read 6 Sent Done Done false
177272024-02-27T21:50:52.367Z 7 Acked 1006 Write 4 Sent Done Done false
177282024-02-27T21:50:52.367Z 8 Acked 1007 Flush 0 Sent Done Done false
177292024-02-27T21:50:52.367Z 9 Acked 1008 Write 7 Sent Done Done false
177302024-02-27T21:50:52.367Z 10 Acked 1009 Write 5 Sent Done Done false
177312024-02-27T21:50:52.367Z 11 Acked 1010 Write 7 Sent Done Done false
177322024-02-27T21:50:52.367Z 12 Acked 1011 Write 1 Sent Done Done false
177332024-02-27T21:50:52.367Z 13 Acked 1012 Write 2 Sent Done Done false
177342024-02-27T21:50:52.367Z 14 Acked 1013 Write 10 Sent Done Done false
177352024-02-27T21:50:52.367Z 15 Acked 1014 Write 7 Sent Done Done false
177362024-02-27T21:50:52.367Z 16 Acked 1015 Read 5 Sent Done Done false
177372024-02-27T21:50:52.367Z 17 Acked 1016 Read 1 Sent Done Done false
177382024-02-27T21:50:52.367Z 18 Acked 1017 Read 10 Sent Done Done false
177392024-02-27T21:50:52.368Z 19 Acked 1018 Read 2 Sent Done Done false
177402024-02-27T21:50:52.368Z 20 Acked 1019 Flush 0 Sent Done Done false
177412024-02-27T21:50:52.368Z 21 Acked 1020 Write 10 Sent Done Done false
177422024-02-27T21:50:52.368Z 22 Acked 1021 Read 6 Sent Done Done false
177432024-02-27T21:50:52.368Z 23 Acked 1022 Write 5 Sent Done Done false
177442024-02-27T21:50:52.368Z 24 Acked 1023 Read 2 Sent Done Done false
177452024-02-27T21:50:52.368Z 25 Acked 1024 Write 9 Sent Done Done false
177462024-02-27T21:50:52.368Z 26 Acked 1025 Flush 0 Sent Done Done false
177472024-02-27T21:50:52.368Z 27 Acked 1026 Write 4 Sent Done Done false
177482024-02-27T21:50:52.368Z 28 Acked 1027 Read 4 Sent Done Done false
177492024-02-27T21:50:52.368Z 29 Acked 1028 Write 3 Sent Done Done false
177502024-02-27T21:50:52.368Z 30 Acked 1029 Read 9 Sent Done Sent false
177512024-02-27T21:50:52.368Z STATES DS:0 DS:1 DS:2 TOTAL
177522024-02-27T21:50:52.369Z New 0 0 0 0
177532024-02-27T21:50:52.369Z Sent 30 0 1 31
177542024-02-27T21:50:52.369Z Done 0 30 29 59
177552024-02-27T21:50:52.369Z Skipped 0 0 0 0
177562024-02-27T21:50:52.369Z Error 0 0 0 0
177572024-02-27T21:50:52.369ZLast Flush: 0 1025 1025
177582024-02-27T21:50:52.369ZDownstairs last five completed:
177592024-02-27T21:50:52.369ZUpstairs last five completed: 30 29 28 27 26
177602024-02-27T21:50:52.369Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:50.726306278Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1806}
177612024-02-27T21:50:52.369ZWrote out file "/tmp/test_repair_verify.data"
177622024-02-27T21:50:52.375Z
177632024-02-27T21:50:52.385ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177642024-02-27T21:50:52.390ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177652024-02-27T21:50:52.390Z 0 000-019 161 161 161 291 291 291 F F F
177662024-02-27T21:50:52.390Z 1 020-039 162 164 164 292 297 297 F F F <---
177672024-02-27T21:50:52.390Z 2 040-059 163 164 164 296 298 298 F F F <---
177682024-02-27T21:50:52.390Z 3 060-079 160 164 164 290 298 298 F F F <---
177692024-02-27T21:50:52.390Z 4 080-099 162 162 162 293 293 293 F F F
177702024-02-27T21:50:52.390Z 5 100-119 161 161 161 291 291 291 F F F
177712024-02-27T21:50:52.390Z 6 120-139 162 162 162 293 293 293 F F F
177722024-02-27T21:50:52.390Z 7 140-159 162 162 162 295 295 295 F F F
177732024-02-27T21:50:52.394Z 8 160-179 162 162 162 292 292 292 F F F
177742024-02-27T21:50:52.394Z 9 180-199 159 164 164 288 298 298 F F F <---
177752024-02-27T21:50:52.394Z 10 200-219 156 164 164 279 298 298 F F F <---
177762024-02-27T21:50:52.394Z 11 220-239 156 164 164 281 297 297 F F F <---
177772024-02-27T21:50:52.394Z 12 240-259 155 155 155 277 277 277 F F F
177782024-02-27T21:50:52.394Z 13 260-279 162 162 162 294 294 294 F F F
177792024-02-27T21:50:52.394Z 14 280-299 162 164 164 294 298 298 F T T <---
177802024-02-27T21:50:52.394Z 15 300-319 163 164 164 296 297 297 F T T <---
177812024-02-27T21:50:52.394Z 16 320-339 158 164 164 286 299 299 F F F <---
177822024-02-27T21:50:52.394Z 17 340-359 162 164 164 294 299 299 F F F <---
177832024-02-27T21:50:52.394Z 18 360-379 160 160 160 289 289 289 F F F
177842024-02-27T21:50:52.394Z 19 380-399 162 162 162 295 295 295 F F F
177852024-02-27T21:50:52.395Z 20 400-419 162 162 162 292 292 292 F F F
177862024-02-27T21:50:52.395Z 21 420-439 163 163 163 296 296 296 F F F
177872024-02-27T21:50:52.395Z 22 440-459 148 148 148 269 269 269 F F F
177882024-02-27T21:50:52.395Z 23 460-479 160 160 160 289 289 289 F F F
177892024-02-27T21:50:52.395Z 24 480-499 163 163 163 296 296 296 F F F
177902024-02-27T21:50:52.395Z 25 500-519 160 160 160 289 289 289 F F F
177912024-02-27T21:50:52.395Z 26 520-539 160 160 160 289 289 289 F F F
177922024-02-27T21:50:52.395Z 27 540-559 161 164 164 291 299 299 F F F <---
177932024-02-27T21:50:52.395Z 28 560-579 156 164 164 281 299 299 F F F <---
177942024-02-27T21:50:52.395Z 29 580-599 160 160 160 290 290 290 F F F
177952024-02-27T21:50:52.395ZMax gen: 164, Max flush: 299
177962024-02-27T21:50:52.395ZError: Difference in extent metadata found!
177972024-02-27T21:50:52.399ZA Difference in extent metadata is expected here
177982024-02-27T21:50:52.399ZOn loop 82
177992024-02-27T21:50:52.399Z
178002024-02-27T21:50:52.399ZVerifying data now
178012024-02-27T21:50:57.030ZNo change after verify
178022024-02-27T21:50:57.030ZLoop: 82 Downstairs dump after verify (and repair):
178032024-02-27T21:50:57.048ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178042024-02-27T21:50:57.048Z 0 000-019 161 161 161 291 291 291 F F F
178052024-02-27T21:50:57.048Z 1 020-039 164 164 164 297 297 297 F F F
178062024-02-27T21:50:57.048Z 2 040-059 164 164 164 298 298 298 F F F
178072024-02-27T21:50:57.048Z 3 060-079 164 164 164 298 298 298 F F F
178082024-02-27T21:50:57.048Z 4 080-099 162 162 162 293 293 293 F F F
178092024-02-27T21:50:57.048Z 5 100-119 161 161 161 291 291 291 F F F
178102024-02-27T21:50:57.048Z 6 120-139 162 162 162 293 293 293 F F F
178112024-02-27T21:50:57.048Z 7 140-159 162 162 162 295 295 295 F F F
178122024-02-27T21:50:57.048Z 8 160-179 162 162 162 292 292 292 F F F
178132024-02-27T21:50:57.048Z 9 180-199 164 164 164 298 298 298 F F F
178142024-02-27T21:50:57.048Z 10 200-219 164 164 164 298 298 298 F F F
178152024-02-27T21:50:57.048Z 11 220-239 164 164 164 297 297 297 F F F
178162024-02-27T21:50:57.048Z 12 240-259 155 155 155 277 277 277 F F F
178172024-02-27T21:50:57.048Z 13 260-279 162 162 162 294 294 294 F F F
178182024-02-27T21:50:57.048Z 14 280-299 165 165 165 300 300 300 F F F
178192024-02-27T21:50:57.048Z 15 300-319 165 165 165 300 300 300 F F F
178202024-02-27T21:50:57.048Z 16 320-339 164 164 164 299 299 299 F F F
178212024-02-27T21:50:57.048Z 17 340-359 164 164 164 299 299 299 F F F
178222024-02-27T21:50:57.048Z 18 360-379 160 160 160 289 289 289 F F F
178232024-02-27T21:50:57.048Z 19 380-399 162 162 162 295 295 295 F F F
178242024-02-27T21:50:57.049Z 20 400-419 162 162 162 292 292 292 F F F
178252024-02-27T21:50:57.049Z 21 420-439 163 163 163 296 296 296 F F F
178262024-02-27T21:50:57.049Z 22 440-459 148 148 148 269 269 269 F F F
178272024-02-27T21:50:57.049Z 23 460-479 160 160 160 289 289 289 F F F
178282024-02-27T21:50:57.049Z 24 480-499 163 163 163 296 296 296 F F F
178292024-02-27T21:50:57.049Z 25 500-519 160 160 160 289 289 289 F F F
178302024-02-27T21:50:57.049Z 26 520-539 160 160 160 289 289 289 F F F
178312024-02-27T21:50:57.049Z 27 540-559 164 164 164 299 299 299 F F F
178322024-02-27T21:50:57.049Z 28 560-579 164 164 164 299 299 299 F F F
178332024-02-27T21:50:57.049Z 29 580-599 160 160 160 290 290 290 F F F
178342024-02-27T21:50:57.049ZMax gen: 165, Max flush: 300
178352024-02-27T21:50:57.049Z
178362024-02-27T21:50:57.049ZBegin loop 83
178372024-02-27T21:50:57.049ZDownstairs to restart: 1
178382024-02-27T21:50:57.060Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.420782165Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814}
178392024-02-27T21:50:57.076Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421131724Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814}
178402024-02-27T21:50:57.077ZCrucible runtime is spawned
178412024-02-27T21:50:57.077Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421185347Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814}
178422024-02-27T21:50:57.077Z{"msg":"Crucible 18b93f4d-ce5a-424e-a770-67d21d99cc98 has session id: b5f9ae97-95e3-4dfb-b949-31c14e5a80ca","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421216704Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178432024-02-27T21:50:57.077Z{"msg":"Upstairs opts: Upstairs UUID: 18b93f4d-ce5a-424e-a770-67d21d99cc98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421263358Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178442024-02-27T21:50:57.077Z{"msg":"Crucible stats registered with UUID: 18b93f4d-ce5a-424e-a770-67d21d99cc98","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421305103Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178452024-02-27T21:50:57.077Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421349368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814}
178462024-02-27T21:50:57.077Z{"msg":"18b93f4d-ce5a-424e-a770-67d21d99cc98 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421485462Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178472024-02-27T21:50:57.077Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421518778Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178482024-02-27T21:50:57.077Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.42158521Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178492024-02-27T21:50:57.077Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421608148Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178502024-02-27T21:50:57.077Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421686828Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178512024-02-27T21:50:57.077Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421721344Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178522024-02-27T21:50:57.077Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421743212Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178532024-02-27T21:50:57.077Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421766289Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178542024-02-27T21:50:57.077Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421809394Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178552024-02-27T21:50:57.077Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421874356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178562024-02-27T21:50:57.077Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.421915171Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178572024-02-27T21:50:57.077Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.422116248Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178582024-02-27T21:50:57.077Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.422149844Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178592024-02-27T21:50:57.077Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.422526429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178602024-02-27T21:50:57.077Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.422600901Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178612024-02-27T21:50:57.077Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.422702499Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178622024-02-27T21:50:57.078Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.422768421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178632024-02-27T21:50:57.078Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.422851701Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178642024-02-27T21:50:57.078Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.422880168Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178652024-02-27T21:50:57.078Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.423044018Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178662024-02-27T21:50:57.078Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.423360201Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178672024-02-27T21:50:57.078Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.423459399Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178682024-02-27T21:50:57.078Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.423547069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178692024-02-27T21:50:57.078Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.423567967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178702024-02-27T21:50:57.078Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.423595833Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178712024-02-27T21:50:57.078Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.423885469Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178722024-02-27T21:50:57.078Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.423949812Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178732024-02-27T21:50:57.078Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.423991927Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178742024-02-27T21:50:57.078Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424009325Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178752024-02-27T21:50:57.078Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424031792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178762024-02-27T21:50:57.078Z{"msg":"[0]R flush_numbers[0..12]: [291, 297, 298, 298, 293, 291, 293, 295, 292, 298, 298, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.42404666Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178772024-02-27T21:50:57.078Z{"msg":"[0]R generation[0..12]: [161, 164, 164, 164, 162, 161, 162, 162, 162, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424067948Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178782024-02-27T21:50:57.078Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424082446Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178792024-02-27T21:50:57.078Z{"msg":"[1]R flush_numbers[0..12]: [291, 297, 298, 298, 293, 291, 293, 295, 292, 298, 298, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424108353Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178802024-02-27T21:50:57.078Z{"msg":"[1]R generation[0..12]: [161, 164, 164, 164, 162, 161, 162, 162, 162, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424122331Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178812024-02-27T21:50:57.078Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424148508Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178822024-02-27T21:50:57.078Z{"msg":"Wait for a query_work_queue command to finish before sending IO
178832024-02-27T21:50:57.078Z[2]R flush_numbers[0..12]: [291, 297, 298, 298, 293, 291, 293, 295, 292, 298, 298, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424173575Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178842024-02-27T21:50:57.078Z{"msg":"[2]R generation[0..12]: [161, 164, 164, 164, 162, 161, 162, 162, 162, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424189273Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178852024-02-27T21:50:57.078Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424203492Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178862024-02-27T21:50:57.078Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.42421731Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178872024-02-27T21:50:57.079Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424233088Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178882024-02-27T21:50:57.079Z{"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424257305Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178892024-02-27T21:50:57.079Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424271834Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"":"downstairs","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178902024-02-27T21:50:57.079Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424295421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178912024-02-27T21:50:57.079Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424329187Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"0","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178922024-02-27T21:50:57.079Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424360303Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"1","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178932024-02-27T21:50:57.079Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424376102Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"client":"2","":"downstairs","session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178942024-02-27T21:50:57.079Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424404028Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178952024-02-27T21:50:57.079Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424438484Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178962024-02-27T21:50:57.079Z{"msg":"18b93f4d-ce5a-424e-a770-67d21d99cc98 is now active with session: b5f9ae97-95e3-4dfb-b949-31c14e5a80ca","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.42447301Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178972024-02-27T21:50:57.079ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178982024-02-27T21:50:57.079Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424512526Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814,"session_id":"b5f9ae97-95e3-4dfb-b949-31c14e5a80ca"}
178992024-02-27T21:50:57.079Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:50:55.424542092Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814}
179002024-02-27T21:50:57.088ZLoading write count information from file "/tmp/test_repair_verify.data"
179012024-02-27T21:50:57.088ZRun Repair workload
179022024-02-27T21:50:57.088Z01/30 Write block 593 len 28672 data: 13 10 9 10 4 5 3
179032024-02-27T21:50:57.088Z02/30 Read block 199 len 4096
179042024-02-27T21:50:57.088Z03/30 Read block 108 len 20480
179052024-02-27T21:50:57.088Z04/30 Read block 346 len 20480
179062024-02-27T21:50:57.088Z05/30 Read block 489 len 4096
179072024-02-27T21:50:57.088Z06/30 Flush
179082024-02-27T21:50:57.088Z07/30 Write block 63 len 24576 data: 11 10 9 9 10 10
179092024-02-27T21:50:57.088Z08/30 Read block 345 len 36864
179102024-02-27T21:50:57.088Z09/30 Write block 111 len 28672 data: 18 16 15 13 12 11 13
179112024-02-27T21:50:57.088Z10/30 Write block 246 len 32768 data: 14 14 13 14 14 15 15 14
179122024-02-27T21:50:57.088Z11/30 Write block 325 len 24576 data: 12 10 12 11 12 12
179132024-02-27T21:50:57.088Z12/30 Read block 204 len 36864
179142024-02-27T21:50:57.088Z13/30 Read block 132 len 20480
179152024-02-27T21:50:57.088Z14/30 Read block 140 len 32768
179162024-02-27T21:50:57.088Z15/30 Read block 135 len 32768
179172024-02-27T21:50:57.088Z16/30 Read block 514 len 36864
179182024-02-27T21:50:57.088Z17/30 Write block 85 len 16384 data: 10 11 14 14
179192024-02-27T21:50:57.089Z18/30 Read block 302 len 36864
179202024-02-27T21:50:57.089Z19/30 Read block 232 len 24576
179212024-02-27T21:50:57.089Z20/30 Read block 434 len 40960
179222024-02-27T21:50:57.089Z21/30 Flush
179232024-02-27T21:50:57.095Z22/30 Write block 553 len 16384 data: 20 20 18 16
179242024-02-27T21:50:57.095Z23/30 Read block 580 len 16384
179252024-02-27T21:50:57.096Z24/30 Read block 495 len 40960
179262024-02-27T21:50:57.096Z25/30 Flush
179272024-02-27T21:50:57.096Z26/30 Flush
179282024-02-27T21:50:57.096Z27/30 Write block 79 len 12288 data: 10 10 10
179292024-02-27T21:50:57.096Z28/30 Write block 139 len 20480 data: 12 12 12 14 12
179302024-02-27T21:50:57.096Z29/30 Write block 318 len 4096 data: 15
179312024-02-27T21:50:57.096Z30/30 Read block 181 len 12288
179322024-02-27T21:50:57.100Z----------------------------------------------------------------
179332024-02-27T21:50:57.100Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
179342024-02-27T21:50:57.100ZStates: Active Active Active
179352024-02-27T21:50:57.100ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179362024-02-27T21:50:57.100Z 1 Acked 1000 Write 7 Done Done Done false
179372024-02-27T21:50:57.101Z 2 Acked 1001 Read 1 Done Done Done false
179382024-02-27T21:50:57.101Z 3 Acked 1002 Read 5 Done Sent Done false
179392024-02-27T21:50:57.101Z 4 Acked 1003 Read 5 Done Sent Done false
179402024-02-27T21:50:57.101Z 5 Acked 1004 Read 1 Done Sent Done false
179412024-02-27T21:50:57.101Z 6 Acked 1005 Flush 0 Done Sent Done false
179422024-02-27T21:50:57.102Z 7 Acked 1006 Write 6 Done Sent Done false
179432024-02-27T21:50:57.102Z 8 Acked 1007 Read 9 Done Sent Done false
179442024-02-27T21:50:57.102Z 9 Acked 1008 Write 7 Done Sent Done false
179452024-02-27T21:50:57.102Z 10 Acked 1009 Write 8 Done Sent Done false
179462024-02-27T21:50:57.102Z 11 Acked 1010 Write 6 Done Sent Done false
179472024-02-27T21:50:57.102Z 12 Acked 1011 Read 9 Done Sent Done false
179482024-02-27T21:50:57.103Z 13 Acked 1012 Read 5 Done Sent Done false
179492024-02-27T21:50:57.103Z 14 Acked 1013 Read 8 Done Sent Done false
179502024-02-27T21:50:57.103Z 15 Acked 1014 Read 8 Done Sent Done false
179512024-02-27T21:50:57.103Z 16 Acked 1015 Read 9 Done Sent Done false
179522024-02-27T21:50:57.103Z 17 Acked 1016 Write 4 Done Sent Done false
179532024-02-27T21:50:57.104Z 18 Acked 1017 Read 9 Done Sent Done false
179542024-02-27T21:50:57.104Z 19 Acked 1018 Read 6 Done Sent Done false
179552024-02-27T21:50:57.104Z 20 Acked 1019 Read 10 Done Sent Done false
179562024-02-27T21:50:57.104Z 21 Acked 1020 Flush 0 Done Sent Done false
179572024-02-27T21:50:57.104Z 22 Acked 1021 Write 4 Done Sent Done false
179582024-02-27T21:50:57.105Z 23 Acked 1022 Read 4 Done Sent Done false
179592024-02-27T21:50:57.105Z 24 Acked 1023 Read 10 Done Sent Done false
179602024-02-27T21:50:57.105Z 25 Acked 1024 Flush 0 Done Sent Done false
179612024-02-27T21:50:57.105Z 26 Acked 1025 Flush 0 Done Sent Done false
179622024-02-27T21:50:57.105Z 27 Acked 1026 Write 3 Done Sent Done false
179632024-02-27T21:50:57.106Z 28 Acked 1027 Write 5 Done Sent Done false
179642024-02-27T21:50:57.106Z 29 Acked 1028 Write 1 Done Sent Sent false
179652024-02-27T21:50:57.106Z 30 Acked 1029 Read 3 Done Sent Sent false
179662024-02-27T21:50:57.106Z STATES DS:0 DS:1 DS:2 TOTAL
179672024-02-27T21:50:57.106Z New 0 0 0 0
179682024-02-27T21:50:57.106Z Sent 0 28 2 30
179692024-02-27T21:50:57.106Z Done 30 2 28 60
179702024-02-27T21:50:57.106Z Skipped 0 0 0 0
179712024-02-27T21:50:57.106Z Error 0 0 0 0
179722024-02-27T21:50:57.107ZLast Flush: 1025 0 1025
179732024-02-27T21:50:57.107ZDownstairs last five completed:
179742024-02-27T21:50:57.107ZUpstairs last five completed: 30 29 28 27 26
179752024-02-27T21:50:57.107Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:50:55.456793393Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1814}
179762024-02-27T21:50:57.107ZWrote out file "/tmp/test_repair_verify.data"
179772024-02-27T21:50:57.107Z
179782024-02-27T21:50:57.119ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179792024-02-27T21:50:57.132ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179802024-02-27T21:50:57.133Z 0 000-019 161 161 161 291 291 291 F F F
179812024-02-27T21:50:57.133Z 1 020-039 164 164 164 297 297 297 F F F
179822024-02-27T21:50:57.133Z 2 040-059 164 164 164 298 298 298 F F F
179832024-02-27T21:50:57.133Z 3 060-079 166 164 166 302 298 302 T F T <---
179842024-02-27T21:50:57.133Z 4 080-099 166 162 166 302 293 302 T F T <---
179852024-02-27T21:50:57.133Z 5 100-119 166 161 166 302 291 302 F F F <---
179862024-02-27T21:50:57.133Z 6 120-139 162 162 162 293 293 293 T F T <---
179872024-02-27T21:50:57.133Z 7 140-159 162 162 162 295 295 295 T F T <---
179882024-02-27T21:50:57.133Z 8 160-179 162 162 162 292 292 292 F F F
179892024-02-27T21:50:57.133Z 9 180-199 164 164 164 298 298 298 F F F
179902024-02-27T21:50:57.133Z 10 200-219 164 164 164 298 298 298 F F F
179912024-02-27T21:50:57.134Z 11 220-239 164 164 164 297 297 297 F F F
179922024-02-27T21:50:57.134Z 12 240-259 166 155 166 302 277 302 F F F <---
179932024-02-27T21:50:57.134Z 13 260-279 162 162 162 294 294 294 F F F
179942024-02-27T21:50:57.134Z 14 280-299 165 165 165 300 300 300 F F F
179952024-02-27T21:50:57.134Z 15 300-319 165 165 165 300 300 300 T F T <---
179962024-02-27T21:50:57.134Z 16 320-339 166 164 166 302 299 302 F F F <---
179972024-02-27T21:50:57.134Z 17 340-359 164 164 164 299 299 299 F F F
179982024-02-27T21:50:57.134Z 18 360-379 160 160 160 289 289 289 F F F
179992024-02-27T21:50:57.134Z 19 380-399 162 162 162 295 295 295 F F F
180002024-02-27T21:50:57.134Z 20 400-419 162 162 162 292 292 292 F F F
180012024-02-27T21:50:57.134Z 21 420-439 163 163 163 296 296 296 F F F
180022024-02-27T21:50:57.134Z 22 440-459 148 148 148 269 269 269 F F F
180032024-02-27T21:50:57.134Z 23 460-479 160 160 160 289 289 289 F F F
180042024-02-27T21:50:57.134Z 24 480-499 163 163 163 296 296 296 F F F
180052024-02-27T21:50:57.135Z 25 500-519 160 160 160 289 289 289 F F F
180062024-02-27T21:50:57.135Z 26 520-539 160 160 160 289 289 289 F F F
180072024-02-27T21:50:57.135Z 27 540-559 166 164 166 303 299 303 F F F <---
180082024-02-27T21:50:57.135Z 28 560-579 164 164 164 299 299 299 F F F
180092024-02-27T21:50:57.135Z 29 580-599 166 160 166 301 290 301 F T F <---
180102024-02-27T21:50:57.135ZMax gen: 166, Max flush: 303
180112024-02-27T21:50:57.135ZError: Difference in extent metadata found!
180122024-02-27T21:50:57.135ZA Difference in extent metadata is expected here
180132024-02-27T21:50:57.135ZOn loop 83
180142024-02-27T21:50:57.135Z
180152024-02-27T21:50:57.138ZVerifying data now
180162024-02-27T21:51:01.663ZNo change after verify
180172024-02-27T21:51:01.663ZLoop: 83 Downstairs dump after verify (and repair):
180182024-02-27T21:51:01.677ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180192024-02-27T21:51:01.677Z 0 000-019 161 161 161 291 291 291 F F F
180202024-02-27T21:51:01.681Z 1 020-039 164 164 164 297 297 297 F F F
180212024-02-27T21:51:01.681Z 2 040-059 164 164 164 298 298 298 F F F
180222024-02-27T21:51:01.681Z 3 060-079 167 167 167 304 304 304 F F F
180232024-02-27T21:51:01.682Z 4 080-099 167 167 167 304 304 304 F F F
180242024-02-27T21:51:01.682Z 5 100-119 166 166 166 302 302 302 F F F
180252024-02-27T21:51:01.682Z 6 120-139 167 167 167 304 304 304 F F F
180262024-02-27T21:51:01.682Z 7 140-159 167 167 167 304 304 304 F F F
180272024-02-27T21:51:01.682Z 8 160-179 162 162 162 292 292 292 F F F
180282024-02-27T21:51:01.682Z 9 180-199 164 164 164 298 298 298 F F F
180292024-02-27T21:51:01.682Z 10 200-219 164 164 164 298 298 298 F F F
180302024-02-27T21:51:01.682Z 11 220-239 164 164 164 297 297 297 F F F
180312024-02-27T21:51:01.682Z 12 240-259 166 166 166 302 302 302 F F F
180322024-02-27T21:51:01.682Z 13 260-279 162 162 162 294 294 294 F F F
180332024-02-27T21:51:01.682Z 14 280-299 165 165 165 300 300 300 F F F
180342024-02-27T21:51:01.682Z 15 300-319 167 167 167 304 304 304 F F F
180352024-02-27T21:51:01.682Z 16 320-339 166 166 166 302 302 302 F F F
180362024-02-27T21:51:01.682Z 17 340-359 164 164 164 299 299 299 F F F
180372024-02-27T21:51:01.682Z 18 360-379 160 160 160 289 289 289 F F F
180382024-02-27T21:51:01.682Z 19 380-399 162 162 162 295 295 295 F F F
180392024-02-27T21:51:01.682Z 20 400-419 162 162 162 292 292 292 F F F
180402024-02-27T21:51:01.682Z 21 420-439 163 163 163 296 296 296 F F F
180412024-02-27T21:51:01.682Z 22 440-459 148 148 148 269 269 269 F F F
180422024-02-27T21:51:01.682Z 23 460-479 160 160 160 289 289 289 F F F
180432024-02-27T21:51:01.682Z 24 480-499 163 163 163 296 296 296 F F F
180442024-02-27T21:51:01.682Z 25 500-519 160 160 160 289 289 289 F F F
180452024-02-27T21:51:01.682Z 26 520-539 160 160 160 289 289 289 F F F
180462024-02-27T21:51:01.682Z 27 540-559 166 166 166 303 303 303 F F F
180472024-02-27T21:51:01.683Z 28 560-579 164 164 164 299 299 299 F F F
180482024-02-27T21:51:01.683Z 29 580-599 166 166 166 301 301 301 F F F
180492024-02-27T21:51:01.683ZMax gen: 167, Max flush: 304
180502024-02-27T21:51:01.683Z
180512024-02-27T21:51:01.683ZBegin loop 84
180522024-02-27T21:51:01.683ZDownstairs to restart: 1
180532024-02-27T21:51:01.699Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.059593168Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822}
180542024-02-27T21:51:01.699Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.059854977Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822}
180552024-02-27T21:51:01.706Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.059893193Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822}
180562024-02-27T21:51:01.706Z{"msg":"Crucible 7686e548-8655-4c33-a69f-e17a3bd4f458 has session id: 5012c514-67b0-4b51-9c78-20a8b887ef37","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.059956515Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180572024-02-27T21:51:01.706ZCrucible runtime is spawned
180582024-02-27T21:51:01.706Z{"msg":"Upstairs opts: Upstairs UUID: 7686e548-8655-4c33-a69f-e17a3bd4f458, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060023067Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180592024-02-27T21:51:01.706Z{"msg":"Crucible stats registered with UUID: 7686e548-8655-4c33-a69f-e17a3bd4f458","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060059783Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180602024-02-27T21:51:01.706Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.06008333Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822}
180612024-02-27T21:51:01.706Z{"msg":"7686e548-8655-4c33-a69f-e17a3bd4f458 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060142923Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180622024-02-27T21:51:01.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060177079Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180632024-02-27T21:51:01.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060225283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180642024-02-27T21:51:01.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.06025518Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180652024-02-27T21:51:01.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060292186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180662024-02-27T21:51:01.706Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060319312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180672024-02-27T21:51:01.706Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060352828Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180682024-02-27T21:51:01.706Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060385464Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180692024-02-27T21:51:01.706Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060432019Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180702024-02-27T21:51:01.706Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060471024Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180712024-02-27T21:51:01.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.06050632Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180722024-02-27T21:51:01.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060532807Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180732024-02-27T21:51:01.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.060634695Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180742024-02-27T21:51:01.706Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.061287218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180752024-02-27T21:51:01.707Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.061328323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180762024-02-27T21:51:01.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.06135726Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180772024-02-27T21:51:01.707Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.061388226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180782024-02-27T21:51:01.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.061492134Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180792024-02-27T21:51:01.707Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.061543818Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180802024-02-27T21:51:01.707Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.061993415Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180812024-02-27T21:51:01.707Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.062041499Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180822024-02-27T21:51:01.707Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.062309538Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180832024-02-27T21:51:01.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.062647398Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180842024-02-27T21:51:01.707Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.062707181Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180852024-02-27T21:51:01.707Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.062748616Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180862024-02-27T21:51:01.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.062771593Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180872024-02-27T21:51:01.707Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.062820688Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180882024-02-27T21:51:01.707Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.062858403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180892024-02-27T21:51:01.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.06288211Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180902024-02-27T21:51:01.707Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.062917256Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180912024-02-27T21:51:01.707Z{"msg":"[0]R flush_numbers[0..12]: [291, 297, 298, 304, 304, 302, 304, 304, 292, 298, 298, 297]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
180922024-02-27T21:51:01.707Z30,"time":"2024-02-27T21:51:00.062958661Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180932024-02-27T21:51:01.707Z{"msg":"[0]R generation[0..12]: [161, 164, 164, 167, 167, 166, 167, 167, 162, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063028953Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180942024-02-27T21:51:01.707Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063069998Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180952024-02-27T21:51:01.707Z{"msg":"[1]R flush_numbers[0..12]: [291, 297, 298, 304, 304, 302, 304, 304, 292, 298, 298, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063108194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180962024-02-27T21:51:01.707Z{"msg":"[1]R generation[0..12]: [161, 164, 164, 167, 167, 166, 167, 167, 162, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063145989Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180972024-02-27T21:51:01.707Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063186025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180982024-02-27T21:51:01.707Z{"msg":"[2]R flush_numbers[0..12]: [291, 297, 298, 304, 304, 302, 304, 304, 292, 298, 298, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063232919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
180992024-02-27T21:51:01.707Z{"msg":"[2]R generation[0..12]: [161, 164, 164, 167, 167, 166, 167, 167, 162, 164, 164, 164]","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
181002024-02-27T21:51:01.708Z,"time":"2024-02-27T21:51:00.063276444Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181012024-02-27T21:51:01.708Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063314739Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181022024-02-27T21:51:01.708Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063332837Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181032024-02-27T21:51:01.708Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063354265Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181042024-02-27T21:51:01.708Z{"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063368663Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181052024-02-27T21:51:01.708Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063389291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"":"downstairs","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181062024-02-27T21:51:01.708Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063403029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181072024-02-27T21:51:01.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063422177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"0","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181082024-02-27T21:51:01.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063436295Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"1","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181092024-02-27T21:51:01.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063456443Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"client":"2","":"downstairs","session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181102024-02-27T21:51:01.712Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063470131Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181112024-02-27T21:51:01.712Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063489069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181122024-02-27T21:51:01.712Z{"msg":"7686e548-8655-4c33-a69f-e17a3bd4f458 is now active with session: 5012c514-67b0-4b51-9c78-20a8b887ef37","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063502377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181132024-02-27T21:51:01.712Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063521195Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822,"session_id":"5012c514-67b0-4b51-9c78-20a8b887ef37"}
181142024-02-27T21:51:01.712Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:00.063533913Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822}
181152024-02-27T21:51:01.712ZLoading write count information from file "/tmp/test_repair_verify.data"
181162024-02-27T21:51:01.712ZRun Repair workload
181172024-02-27T21:51:01.712Z01/30 Write block 153 len 32768 data: 10 10 13 14 14 13 14 14
181182024-02-27T21:51:01.712Z02/30 Write block 585 len 24576 data: 14 14 12 12 13 14
181192024-02-27T21:51:01.712Z03/30 Read block 100 len 24576
181202024-02-27T21:51:01.712Z04/30 Write block 383 len 16384 data: 12 13 14 11
181212024-02-27T21:51:01.712Z05/30 Write block 167 len 12288 data: 9 8 10
181222024-02-27T21:51:01.712Z06/30 Read block 338 len 20480
181232024-02-27T21:51:01.712Z07/30 Read block 467 len 40960
181242024-02-27T21:51:01.712Z08/30 Write block 415 len 40960 data: 14 15 19 17 16 11 9 10 11 11
181252024-02-27T21:51:01.712Z09/30 Write block 482 len 4096 data: 14
181262024-02-27T21:51:01.713Z10/30 Read block 107 len 4096
181272024-02-27T21:51:01.724Z11/30 Write block 172 len 12288 data: 10 9 13
181282024-02-27T21:51:01.724Z12/30 Read block 241 len 16384
181292024-02-27T21:51:01.724Z13/30 Write block 429 len 12288 data: 10 12 12
181302024-02-27T21:51:01.724Z14/30 Read block 384 len 36864
181312024-02-27T21:51:01.724Z15/30 Write block 432 len 28672 data: 12 12 14 11 8 9 8
181322024-02-27T21:51:01.724Z16/30 Read block 414 len 40960
181332024-02-27T21:51:01.724Z17/30 Read block 491 len 32768
181342024-02-27T21:51:01.724Z18/30 Write block 409 len 4096 data: 10
181352024-02-27T21:51:01.724Z19/30 Write block 296 len 8192 data: 17 17
181362024-02-27T21:51:01.724Z20/30 Read block 324 len 16384
181372024-02-27T21:51:01.724Z21/30 Read block 378 len 4096
181382024-02-27T21:51:01.724Z22/30 Read block 501 len 24576
181392024-02-27T21:51:01.727Z23/30 Write block 522 len 40960 data: 12 10 11 10 12 13 11 9 8 8
181402024-02-27T21:51:01.727Z24/30 Read block 209 len 32768
181412024-02-27T21:51:01.728Z25/30 Read block 274 len 40960
181422024-02-27T21:51:01.728Z26/30 Write block 441 len 36864 data: 6 7 8 6 6 7 7 9 9
181432024-02-27T21:51:01.728Z27/30 Write block 285 len 4096 data: 6
181442024-02-27T21:51:01.728Z28/30 Write block 529 len 4096 data: 10
181452024-02-27T21:51:01.728Z29/30 Read block 292 len 40960
181462024-02-27T21:51:01.734Z30/30 Write block 91 len 36864 data: 12 9 13 12 13 12 12 11 15
181472024-02-27T21:51:01.735Z----------------------------------------------------------------
181482024-02-27T21:51:01.735Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
181492024-02-27T21:51:01.735ZStates: Active Active Active
181502024-02-27T21:51:01.735ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
181512024-02-27T21:51:01.735Z 1 Acked 1000 Write 8 Done Done Done false
181522024-02-27T21:51:01.735Z 2 Acked 1001 Write 6 Done Done Done false
181532024-02-27T21:51:01.735Z 3 Acked 1002 Read 6 Done Done Done false
181542024-02-27T21:51:01.735Z 4 Acked 1003 Write 4 Done Sent Done false
181552024-02-27T21:51:01.735Z 5 Acked 1004 Write 3 Done Sent Done false
181562024-02-27T21:51:01.735Z 6 Acked 1005 Read 5 Done Sent Done false
181572024-02-27T21:51:01.735Z 7 Acked 1006 Read 10 Done Sent Done false
181582024-02-27T21:51:01.735Z 8 Acked 1007 Write 10 Done Sent Done false
181592024-02-27T21:51:01.735Z 9 Acked 1008 Write 1 Done Sent Done false
181602024-02-27T21:51:01.735Z 10 Acked 1009 Read 1 Done Sent Done false
181612024-02-27T21:51:01.735Z 11 Acked 1010 Write 3 Done Sent Done false
181622024-02-27T21:51:01.735Z 12 Acked 1011 Read 4 Done Sent Done false
181632024-02-27T21:51:01.735Z 13 Acked 1012 Write 3 Done Sent Done false
181642024-02-27T21:51:01.735Z 14 Acked 1013 Read 9 Done Sent Done false
181652024-02-27T21:51:01.735Z 15 Acked 1014 Write 7 Done Sent Done false
181662024-02-27T21:51:01.735Z 16 Acked 1015 Read 10 Done Sent Done false
181672024-02-27T21:51:01.735Z 17 Acked 1016 Read 8 Done Sent Done false
181682024-02-27T21:51:01.735Z 18 Acked 1017 Write 1 Done Sent Done false
181692024-02-27T21:51:01.735Z 19 Acked 1018 Write 2 Done Sent Done false
181702024-02-27T21:51:01.736Z 20 Acked 1019 Read 4 Done Sent Done false
181712024-02-27T21:51:01.736Z 21 Acked 1020 Read 1 Done Sent Done false
181722024-02-27T21:51:01.736Z 22 Acked 1021 Read 6 Done Sent Done false
181732024-02-27T21:51:01.736Z 23 Acked 1022 Write 10 Done Sent Done false
181742024-02-27T21:51:01.736Z 24 Acked 1023 Read 8 Done Sent Done false
181752024-02-27T21:51:01.736Z 25 Acked 1024 Read 10 Done Sent Done false
181762024-02-27T21:51:01.736Z 26 Acked 1025 Write 9 Done Sent Done false
181772024-02-27T21:51:01.736Z 27 Acked 1026 Write 1 Done Sent Done false
181782024-02-27T21:51:01.736Z 28 Acked 1027 Write 1 Done Sent Done false
181792024-02-27T21:51:01.736Z 29 Acked 1028 Read 10 Done Sent Done false
181802024-02-27T21:51:01.736Z 30 Acked 1029 Write 9 Sent Sent Sent false
181812024-02-27T21:51:01.736Z STATES DS:0 DS:1 DS:2 TOTAL
181822024-02-27T21:51:01.736Z New 0 0 0 0
181832024-02-27T21:51:01.736Z Sent 1 27 1 29
181842024-02-27T21:51:01.736Z Done 29 3 29 61
181852024-02-27T21:51:01.736Z Skipped 0 0 0 0
181862024-02-27T21:51:01.736Z Error 0 0 0 0
181872024-02-27T21:51:01.736ZLast Flush: 0 0 0
181882024-02-27T21:51:01.736ZDownstairs last five completed:
181892024-02-27T21:51:01.736ZUpstairs last five completed: 30 29 28 27 26
181902024-02-27T21:51:01.736Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:00.090460292Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1822}
181912024-02-27T21:51:01.736ZWrote out file "/tmp/test_repair_verify.data"
181922024-02-27T21:51:01.740Z
181932024-02-27T21:51:01.740ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181942024-02-27T21:51:01.759ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181952024-02-27T21:51:01.759Z 0 000-019 161 161 161 291 291 291 F F F
181962024-02-27T21:51:01.759Z 1 020-039 164 164 164 297 297 297 F F F
181972024-02-27T21:51:01.759Z 2 040-059 164 164 164 298 298 298 F F F
181982024-02-27T21:51:01.759Z 3 060-079 167 167 167 304 304 304 F F F
181992024-02-27T21:51:01.759Z 4 080-099 167 167 167 304 304 304 T F T <---
182002024-02-27T21:51:01.759Z 5 100-119 166 166 166 302 302 302 F F F
182012024-02-27T21:51:01.759Z 6 120-139 167 167 167 304 304 304 F F F
182022024-02-27T21:51:01.759Z 7 140-159 167 167 167 304 304 304 T T T
182032024-02-27T21:51:01.759Z 8 160-179 162 162 162 292 292 292 T T T
182042024-02-27T21:51:01.763Z 9 180-199 164 164 164 298 298 298 F F F
182052024-02-27T21:51:01.763Z 10 200-219 164 164 164 298 298 298 F F F
182062024-02-27T21:51:01.763Z 11 220-239 164 164 164 297 297 297 F F F
182072024-02-27T21:51:01.763Z 12 240-259 166 166 166 302 302 302 F F F
182082024-02-27T21:51:01.763Z 13 260-279 162 162 162 294 294 294 F F F
182092024-02-27T21:51:01.763Z 14 280-299 165 165 165 300 300 300 T F T <---
182102024-02-27T21:51:01.763Z 15 300-319 167 167 167 304 304 304 F F F
182112024-02-27T21:51:01.763Z 16 320-339 166 166 166 302 302 302 F F F
182122024-02-27T21:51:01.763Z 17 340-359 164 164 164 299 299 299 F F F
182132024-02-27T21:51:01.763Z 18 360-379 160 160 160 289 289 289 F F F
182142024-02-27T21:51:01.763Z 19 380-399 162 162 162 295 295 295 T F T <---
182152024-02-27T21:51:01.763Z 20 400-419 162 162 162 292 292 292 T F T <---
182162024-02-27T21:51:01.763Z 21 420-439 163 163 163 296 296 296 T F T <---
182172024-02-27T21:51:01.764Z 22 440-459 148 148 148 269 269 269 T F T <---
182182024-02-27T21:51:01.764Z 23 460-479 160 160 160 289 289 289 F F F
182192024-02-27T21:51:01.764Z 24 480-499 163 163 163 296 296 296 T F T <---
182202024-02-27T21:51:01.764Z 25 500-519 160 160 160 289 289 289 F F F
182212024-02-27T21:51:01.764Z 26 520-539 160 160 160 289 289 289 T F T <---
182222024-02-27T21:51:01.764Z 27 540-559 166 166 166 303 303 303 F F F
182232024-02-27T21:51:01.764Z 28 560-579 164 164 164 299 299 299 F F F
182242024-02-27T21:51:01.764Z 29 580-599 166 166 166 301 301 301 T T T
182252024-02-27T21:51:01.764ZMax gen: 167, Max flush: 304
182262024-02-27T21:51:01.764ZError: Difference in extent metadata found!
182272024-02-27T21:51:01.764ZA Difference in extent metadata is expected here
182282024-02-27T21:51:01.764ZOn loop 84
182292024-02-27T21:51:01.764Z
182302024-02-27T21:51:01.767ZVerifying data now
182312024-02-27T21:51:07.357ZNo change after verify
182322024-02-27T21:51:07.357ZLoop: 84 Downstairs dump after verify (and repair):
182332024-02-27T21:51:07.373ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182342024-02-27T21:51:07.373Z 0 000-019 161 161 161 291 291 291 F F F
182352024-02-27T21:51:07.373Z 1 020-039 164 164 164 297 297 297 F F F
182362024-02-27T21:51:07.373Z 2 040-059 164 164 164 298 298 298 F F F
182372024-02-27T21:51:07.373Z 3 060-079 167 167 167 304 304 304 F F F
182382024-02-27T21:51:07.373Z 4 080-099 168 168 168 305 305 305 F F F
182392024-02-27T21:51:07.373Z 5 100-119 166 166 166 302 302 302 F F F
182402024-02-27T21:51:07.373Z 6 120-139 167 167 167 304 304 304 F F F
182412024-02-27T21:51:07.373Z 7 140-159 168 168 168 305 305 305 F F F
182422024-02-27T21:51:07.373Z 8 160-179 168 168 168 305 305 305 F F F
182432024-02-27T21:51:07.373Z 9 180-199 164 164 164 298 298 298 F F F
182442024-02-27T21:51:07.373Z 10 200-219 164 164 164 298 298 298 F F F
182452024-02-27T21:51:07.373Z 11 220-239 164 164 164 297 297 297 F F F
182462024-02-27T21:51:07.373Z 12 240-259 166 166 166 302 302 302 F F F
182472024-02-27T21:51:07.378Z 13 260-279 162 162 162 294 294 294 F F F
182482024-02-27T21:51:07.378Z 14 280-299 168 168 168 305 305 305 F F F
182492024-02-27T21:51:07.378Z 15 300-319 167 167 167 304 304 304 F F F
182502024-02-27T21:51:07.378Z 16 320-339 166 166 166 302 302 302 F F F
182512024-02-27T21:51:07.378Z 17 340-359 164 164 164 299 299 299 F F F
182522024-02-27T21:51:07.378Z 18 360-379 160 160 160 289 289 289 F F F
182532024-02-27T21:51:07.378Z 19 380-399 168 168 168 305 305 305 F F F
182542024-02-27T21:51:07.378Z 20 400-419 168 168 168 305 305 305 F F F
182552024-02-27T21:51:07.378Z 21 420-439 168 168 168 305 305 305 F F F
182562024-02-27T21:51:07.378Z 22 440-459 168 168 168 305 305 305 F F F
182572024-02-27T21:51:07.378Z 23 460-479 160 160 160 289 289 289 F F F
182582024-02-27T21:51:07.378Z 24 480-499 168 168 168 305 305 305 F F F
182592024-02-27T21:51:07.378Z 25 500-519 160 160 160 289 289 289 F F F
182602024-02-27T21:51:07.378Z 26 520-539 168 168 168 305 305 305 F F F
182612024-02-27T21:51:07.378Z 27 540-559 166 166 166 303 303 303 F F F
182622024-02-27T21:51:07.378Z 28 560-579 164 164 164 299 299 299 F F F
182632024-02-27T21:51:07.378Z 29 580-599 168 168 168 305 305 305 F F F
182642024-02-27T21:51:07.378ZMax gen: 168, Max flush: 305
182652024-02-27T21:51:07.379Z
182662024-02-27T21:51:07.379ZBegin loop 85
182672024-02-27T21:51:07.379ZDownstairs to restart: 1
182682024-02-27T21:51:07.396Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.755441101Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830}
182692024-02-27T21:51:07.396Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.755747134Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830}
182702024-02-27T21:51:07.403Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.75578485Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830}
182712024-02-27T21:51:07.403Z{"msg":"Crucible c0edade2-70d0-47ac-8b0f-b5c05a5e612a has session id: e9224fca-2987-4f88-abe4-0fc22683b382","v":0,"name":"crucible","level":30Crucible runtime is spawned
182722024-02-27T21:51:07.403Z,"time":"2024-02-27T21:51:05.755820806Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182732024-02-27T21:51:07.403Z{"msg":"Upstairs opts: Upstairs UUID: c0edade2-70d0-47ac-8b0f-b5c05a5e612a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.755909346Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182742024-02-27T21:51:07.403Z{"msg":"Crucible stats registered with UUID: c0edade2-70d0-47ac-8b0f-b5c05a5e612a","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.755949151Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182752024-02-27T21:51:07.404Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.755968388Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830}
182762024-02-27T21:51:07.404Z{"msg":"c0edade2-70d0-47ac-8b0f-b5c05a5e612a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756082965Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182772024-02-27T21:51:07.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756129229Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182782024-02-27T21:51:07.404Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756157606Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182792024-02-27T21:51:07.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756189622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182802024-02-27T21:51:07.404Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.75621327Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182812024-02-27T21:51:07.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756245916Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182822024-02-27T21:51:07.404Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756272343Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182832024-02-27T21:51:07.404Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.7562971Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182842024-02-27T21:51:07.404Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756328096Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182852024-02-27T21:51:07.404Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756355353Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182862024-02-27T21:51:07.404Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756455661Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182872024-02-27T21:51:07.404Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756516694Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182882024-02-27T21:51:07.404Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.756567198Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182892024-02-27T21:51:07.404Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.75714595Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182902024-02-27T21:51:07.404Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.757188135Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182912024-02-27T21:51:07.404Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.757238769Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182922024-02-27T21:51:07.404Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.757291643Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182932024-02-27T21:51:07.404Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.757320709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182942024-02-27T21:51:07.404Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.757345946Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182952024-02-27T21:51:07.404Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.757766917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182962024-02-27T21:51:07.404Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.757810791Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182972024-02-27T21:51:07.405Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.757977782Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182982024-02-27T21:51:07.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758407811Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
182992024-02-27T21:51:07.405Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758463015Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183002024-02-27T21:51:07.405Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.75850082Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183012024-02-27T21:51:07.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758576361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183022024-02-27T21:51:07.405Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758605568Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183032024-02-27T21:51:07.405Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758623126Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183042024-02-27T21:51:07.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758651792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183052024-02-27T21:51:07.405Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.75867229Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183062024-02-27T21:51:07.405Z{"msg":"[0]R flush_numbers[0..12]: [291, 297, 298, 304, 305, 302, 304, 305, 305, 298, 298, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758701606Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183072024-02-27T21:51:07.405Z{"msg":"[0]R generation[0..12]: [161, 164, 164, 167, 168, 166, 167, 168, 168, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758731593Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183082024-02-27T21:51:07.405Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758769389Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183092024-02-27T21:51:07.405ZWait for a query_work_queue command to finish before sending IO
183102024-02-27T21:51:07.405Z{"msg":"[1]R flush_numbers[0..12]: [291, 297, 298, 304, 305, 302, 304, 305, 305, 298, 298, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758803745Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183112024-02-27T21:51:07.405Z{"msg":"[1]R generation[0..12]: [161, 164, 164, 167, 168, 166, 167, 168, 168, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758823662Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183122024-02-27T21:51:07.405Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758854368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183132024-02-27T21:51:07.405Z{"msg":"[2]R flush_numbers[0..12]: [291, 297, 298, 304, 305, 302, 304, 305, 305, 298, 298, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758874486Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183142024-02-27T21:51:07.405Z{"msg":"[2]R generation[0..12]: [161, 164, 164, 167, 168, 166, 167, 168, 168, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758904323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183152024-02-27T21:51:07.405Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758920211Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183162024-02-27T21:51:07.405Z{"msg":"Max found gen is 169","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758951107Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183172024-02-27T21:51:07.405Z{"msg":"Generation requested: 170 >= found:169","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.758970085Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183182024-02-27T21:51:07.405Z{"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759017169Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183192024-02-27T21:51:07.405Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759050935Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"":"downstairs","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183202024-02-27T21:51:07.405Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759069403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183212024-02-27T21:51:07.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.75909741Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"0","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183222024-02-27T21:51:07.406Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759112378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"1","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183232024-02-27T21:51:07.406Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759142765Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"client":"2","":"downstairs","session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183242024-02-27T21:51:07.406Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759164622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183252024-02-27T21:51:07.406Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759194359Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183262024-02-27T21:51:07.406Z{"msg":"c0edade2-70d0-47ac-8b0f-b5c05a5e612a is now active with session: e9224fca-2987-4f88-abe4-0fc22683b382","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759208767Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183272024-02-27T21:51:07.406Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759221435Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830,"session_id":"e9224fca-2987-4f88-abe4-0fc22683b382"}
183282024-02-27T21:51:07.406Z{"msg":"The guest has finished waiting for activation with:170"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
183292024-02-27T21:51:07.406Z,"v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:05.759259821Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830}
183302024-02-27T21:51:07.409ZLoading write count information from file "/tmp/test_repair_verify.data"
183312024-02-27T21:51:07.409ZRun Repair workload
183322024-02-27T21:51:07.410Z01/30 Write block 220 len 20480 data: 13 12 14 12 12
183332024-02-27T21:51:07.410Z02/30 Read block 385 len 20480
183342024-02-27T21:51:07.410Z03/30 Read block 150 len 20480
183352024-02-27T21:51:07.410Z04/30 Write block 264 len 4096 data: 16
183362024-02-27T21:51:07.410Z05/30 Read block 48 len 24576
183372024-02-27T21:51:07.410Z06/30 Flush
183382024-02-27T21:51:07.410Z07/30 Write block 114 len 4096 data: 14
183392024-02-27T21:51:07.410Z08/30 Write block 267 len 28672 data: 9 11 11 12 13 14 10
183402024-02-27T21:51:07.410Z09/30 Read block 263 len 32768
183412024-02-27T21:51:07.410Z10/30 Read block 27 len 24576
183422024-02-27T21:51:07.416Z11/30 Write block 515 len 32768 data: 12 13 16 13 12 15 12 13
183432024-02-27T21:51:07.416Z12/30 Read block 209 len 40960
183442024-02-27T21:51:07.416Z13/30 Read block 400 len 28672
183452024-02-27T21:51:07.419Z14/30 Flush
183462024-02-27T21:51:07.419Z15/30 Write block 374 len 28672 data: 8 8 8 10 6 8 9
183472024-02-27T21:51:07.419Z16/30 Write block 291 len 20480 data: 16 15 19 18 19
183482024-02-27T21:51:07.419Z17/30 Write block 225 len 12288 data: 11 11 13
183492024-02-27T21:51:07.419Z18/30 Flush
183502024-02-27T21:51:07.423Z19/30 Write block 97 len 12288 data: 13 12 16
183512024-02-27T21:51:07.423Z20/30 Flush
183522024-02-27T21:51:07.426Z21/30 Write block 197 len 28672 data: 11 9 11 12 10 9 9
183532024-02-27T21:51:07.426Z22/30 Write block 272 len 24576 data: 15 11 11 10 12 11
183542024-02-27T21:51:07.426Z23/30 Write block 30 len 40960 data: 13 12 11 11 12 11 11 8 8 10
183552024-02-27T21:51:07.430Z24/30 Write block 314 len 4096 data: 14
183562024-02-27T21:51:07.430Z25/30 Flush
183572024-02-27T21:51:07.430Z26/30 Write block 170 len 32768 data: 11 10 11 10 14 12 14 12
183582024-02-27T21:51:07.436Z27/30 Read block 427 len 4096
183592024-02-27T21:51:07.436Z28/30 Write block 424 len 24576 data: 12 7 7 8 10 11
183602024-02-27T21:51:07.436Z29/30 Read block 342 len 32768
183612024-02-27T21:51:07.436Z30/30 Read block 573 len 4096
183622024-02-27T21:51:07.436Z----------------------------------------------------------------
183632024-02-27T21:51:07.436Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
183642024-02-27T21:51:07.436ZStates: Active Active Active
183652024-02-27T21:51:07.436ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
183662024-02-27T21:51:07.436Z 1 Acked 1000 Write 5 Done Done Done false
183672024-02-27T21:51:07.437Z 2 Acked 1001 Read 5 Done Done Done false
183682024-02-27T21:51:07.437Z 3 Acked 1002 Read 5 Done Sent Done false
183692024-02-27T21:51:07.437Z 4 Acked 1003 Write 1 Done Sent Done false
183702024-02-27T21:51:07.437Z 5 Acked 1004 Read 6 Done Sent Done false
183712024-02-27T21:51:07.437Z 6 Acked 1005 Flush 0 Done Sent Done false
183722024-02-27T21:51:07.437Z 7 Acked 1006 Write 1 Done Sent Done false
183732024-02-27T21:51:07.437Z 8 Acked 1007 Write 7 Done Sent Done false
183742024-02-27T21:51:07.437Z 9 Acked 1008 Read 8 Done Sent Done false
183752024-02-27T21:51:07.437Z 10 Acked 1009 Read 6 Done Sent Done false
183762024-02-27T21:51:07.437Z 11 Acked 1010 Write 8 Done Sent Done false
183772024-02-27T21:51:07.437Z 12 Acked 1011 Read 10 Done Sent Done false
183782024-02-27T21:51:07.437Z 13 Acked 1012 Read 7 Done Sent Done false
183792024-02-27T21:51:07.437Z 14 Acked 1013 Flush 0 Done Sent Done false
183802024-02-27T21:51:07.437Z 15 Acked 1014 Write 7 Done Sent Done false
183812024-02-27T21:51:07.437Z 16 Acked 1015 Write 5 Done Sent Done false
183822024-02-27T21:51:07.437Z 17 Acked 1016 Write 3 Done Sent Done false
183832024-02-27T21:51:07.437Z 18 Acked 1017 Flush 0 Done Sent Done false
183842024-02-27T21:51:07.437Z 19 Acked 1018 Write 3 Done Sent Done false
183852024-02-27T21:51:07.437Z 20 Acked 1019 Flush 0 Done Sent Done false
183862024-02-27T21:51:07.437Z 21 Acked 1020 Write 7 Done Sent Done false
183872024-02-27T21:51:07.437Z 22 Acked 1021 Write 6 Done Sent Done false
183882024-02-27T21:51:07.437Z 23 Acked 1022 Write 10 Done Sent Done false
183892024-02-27T21:51:07.437Z 24 Acked 1023 Write 1 Done Sent Done false
183902024-02-27T21:51:07.437Z 25 Acked 1024 Flush 0 Done Sent Done false
183912024-02-27T21:51:07.438Z 26 Acked 1025 Write 8 Done Sent Done false
183922024-02-27T21:51:07.438Z 27 Acked 1026 Read 1 Done Sent Done false
183932024-02-27T21:51:07.438Z 28 Acked 1027 Write 6 Done Sent Done false
183942024-02-27T21:51:07.438Z 29 Acked 1028 Read 8 Done Sent Done false
183952024-02-27T21:51:07.438Z 30 Acked 1029 Read 1 Done Sent Done false
183962024-02-27T21:51:07.438Z STATES DS:0 DS:1 DS:2 TOTAL
183972024-02-27T21:51:07.438Z New 0 0 0 0
183982024-02-27T21:51:07.438Z Sent 0 28 0 28
183992024-02-27T21:51:07.438Z Done 30 2 30 62
184002024-02-27T21:51:07.438Z Skipped 0 0 0 0
184012024-02-27T21:51:07.438Z Error 0 0 0 0
184022024-02-27T21:51:07.438ZLast Flush: 1024 0 1024
184032024-02-27T21:51:07.438ZDownstairs last five completed:
184042024-02-27T21:51:07.438ZUpstairs last five completed: 30 29 28 27 26
184052024-02-27T21:51:07.438Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:05.7939895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1830}
184062024-02-27T21:51:07.438ZWrote out file "/tmp/test_repair_verify.data"
184072024-02-27T21:51:07.443Z
184082024-02-27T21:51:07.446ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
184092024-02-27T21:51:07.457ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184102024-02-27T21:51:07.457Z 0 000-019 161 161 161 291 291 291 F F F
184112024-02-27T21:51:07.457Z 1 020-039 170 164 170 310 297 310 F F F <---
184122024-02-27T21:51:07.457Z 2 040-059 164 164 164 298 298 298 F F F
184132024-02-27T21:51:07.457Z 3 060-079 167 167 167 304 304 304 F F F
184142024-02-27T21:51:07.457Z 4 080-099 170 168 170 309 305 309 F F F <---
184152024-02-27T21:51:07.457Z 5 100-119 170 166 170 307 302 307 F F F <---
184162024-02-27T21:51:07.461Z 6 120-139 167 167 167 304 304 304 F F F
184172024-02-27T21:51:07.461Z 7 140-159 168 168 168 305 305 305 F F F
184182024-02-27T21:51:07.461Z 8 160-179 168 168 168 305 305 305 T F T <---
184192024-02-27T21:51:07.461Z 9 180-199 170 164 170 310 298 310 F F F <---
184202024-02-27T21:51:07.461Z 10 200-219 170 164 170 310 298 310 F F F <---
184212024-02-27T21:51:07.461Z 11 220-239 170 164 170 308 297 308 F T F <---
184222024-02-27T21:51:07.461Z 12 240-259 166 166 166 302 302 302 F F F
184232024-02-27T21:51:07.461Z 13 260-279 170 162 170 310 294 310 F F F <---
184242024-02-27T21:51:07.461Z 14 280-299 170 168 170 308 305 308 F F F <---
184252024-02-27T21:51:07.461Z 15 300-319 170 167 170 310 304 310 F F F <---
184262024-02-27T21:51:07.461Z 16 320-339 166 166 166 302 302 302 F F F
184272024-02-27T21:51:07.461Z 17 340-359 164 164 164 299 299 299 F F F
184282024-02-27T21:51:07.461Z 18 360-379 170 160 170 308 289 308 F F F <---
184292024-02-27T21:51:07.461Z 19 380-399 170 168 170 308 305 308 F F F <---
184302024-02-27T21:51:07.461Z 20 400-419 168 168 168 305 305 305 F F F
184312024-02-27T21:51:07.461Z 21 420-439 168 168 168 305 305 305 T F T <---
184322024-02-27T21:51:07.461Z 22 440-459 168 168 168 305 305 305 F F F
184332024-02-27T21:51:07.461Z 23 460-479 160 160 160 289 289 289 F F F
184342024-02-27T21:51:07.462Z 24 480-499 168 168 168 305 305 305 F F F
184352024-02-27T21:51:07.462Z 25 500-519 170 160 170 307 289 307 F F F <---
184362024-02-27T21:51:07.462Z 26 520-539 170 168 170 307 305 307 F F F <---
184372024-02-27T21:51:07.462Z 27 540-559 166 166 166 303 303 303 F F F
184382024-02-27T21:51:07.462Z 28 560-579 164 164 164 299 299 299 F F F
184392024-02-27T21:51:07.462Z 29 580-599 168 168 168 305 305 305 F F F
184402024-02-27T21:51:07.462ZMax gen: 170, Max flush: 310
184412024-02-27T21:51:07.462ZError: Difference in extent metadata found!
184422024-02-27T21:51:07.462ZA Difference in extent metadata is expected here
184432024-02-27T21:51:07.462ZOn loop 85
184442024-02-27T21:51:07.462Z
184452024-02-27T21:51:07.465ZVerifying data now
184462024-02-27T21:51:12.194ZNo change after verify
184472024-02-27T21:51:12.194ZLoop: 85 Downstairs dump after verify (and repair):
184482024-02-27T21:51:12.208ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184492024-02-27T21:51:12.208Z 0 000-019 161 161 161 291 291 291 F F F
184502024-02-27T21:51:12.208Z 1 020-039 170 170 170 310 310 310 F F F
184512024-02-27T21:51:12.208Z 2 040-059 164 164 164 298 298 298 F F F
184522024-02-27T21:51:12.208Z 3 060-079 167 167 167 304 304 304 F F F
184532024-02-27T21:51:12.208Z 4 080-099 170 170 170 309 309 309 F F F
184542024-02-27T21:51:12.208Z 5 100-119 170 170 170 307 307 307 F F F
184552024-02-27T21:51:12.208Z 6 120-139 167 167 167 304 304 304 F F F
184562024-02-27T21:51:12.208Z 7 140-159 168 168 168 305 305 305 F F F
184572024-02-27T21:51:12.209Z 8 160-179 171 171 171 311 311 311 F F F
184582024-02-27T21:51:12.212Z 9 180-199 170 170 170 310 310 310 F F F
184592024-02-27T21:51:12.212Z 10 200-219 170 170 170 310 310 310 F F F
184602024-02-27T21:51:12.212Z 11 220-239 170 170 170 308 308 308 F F F
184612024-02-27T21:51:12.212Z 12 240-259 166 166 166 302 302 302 F F F
184622024-02-27T21:51:12.212Z 13 260-279 170 170 170 310 310 310 F F F
184632024-02-27T21:51:12.212Z 14 280-299 170 170 170 308 308 308 F F F
184642024-02-27T21:51:12.212Z 15 300-319 170 170 170 310 310 310 F F F
184652024-02-27T21:51:12.212Z 16 320-339 166 166 166 302 302 302 F F F
184662024-02-27T21:51:12.212Z 17 340-359 164 164 164 299 299 299 F F F
184672024-02-27T21:51:12.212Z 18 360-379 170 170 170 308 308 308 F F F
184682024-02-27T21:51:12.212Z 19 380-399 170 170 170 308 308 308 F F F
184692024-02-27T21:51:12.213Z 20 400-419 168 168 168 305 305 305 F F F
184702024-02-27T21:51:12.213Z 21 420-439 171 171 171 311 311 311 F F F
184712024-02-27T21:51:12.213Z 22 440-459 168 168 168 305 305 305 F F F
184722024-02-27T21:51:12.213Z 23 460-479 160 160 160 289 289 289 F F F
184732024-02-27T21:51:12.213Z 24 480-499 168 168 168 305 305 305 F F F
184742024-02-27T21:51:12.213Z 25 500-519 170 170 170 307 307 307 F F F
184752024-02-27T21:51:12.213Z 26 520-539 170 170 170 307 307 307 F F F
184762024-02-27T21:51:12.213Z 27 540-559 166 166 166 303 303 303 F F F
184772024-02-27T21:51:12.213Z 28 560-579 164 164 164 299 299 299 F F F
184782024-02-27T21:51:12.213Z 29 580-599 168 168 168 305 305 305 F F F
184792024-02-27T21:51:12.213ZMax gen: 171, Max flush: 311
184802024-02-27T21:51:12.213Z
184812024-02-27T21:51:12.213ZBegin loop 86
184822024-02-27T21:51:12.213ZDownstairs to restart: 1
184832024-02-27T21:51:12.231Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.588825935Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838}
184842024-02-27T21:51:12.231Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589100383Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838}
184852024-02-27T21:51:12.238Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589136699Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838}
184862024-02-27T21:51:12.238Z{"msg":"Crucible 4eed15ac-1696-4df3-a8bc-aada3b32510e has session id: c0d67474-0aa4-4ab1-b230-6b2c2ad183d3","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589161436Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184872024-02-27T21:51:12.238Z{"msg":"Upstairs opts: Upstairs UUID: 4eed15ac-1696-4df3-a8bc-aada3b32510e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589208121Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184882024-02-27T21:51:12.238Z{"msg":"Crucible stats registered with UUID: 4eed15ac-1696-4df3-a8bc-aada3b32510e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589247356Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184892024-02-27T21:51:12.238ZCrucible runtime is spawned
184902024-02-27T21:51:12.238Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.58938457Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838}
184912024-02-27T21:51:12.238Z{"msg":"4eed15ac-1696-4df3-a8bc-aada3b32510e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589486388Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184922024-02-27T21:51:12.238Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589510735Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184932024-02-27T21:51:12.238Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589531483Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184942024-02-27T21:51:12.238Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589559739Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184952024-02-27T21:51:12.238Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589574577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184962024-02-27T21:51:12.239Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589588716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184972024-02-27T21:51:12.239Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589606974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184982024-02-27T21:51:12.239Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589631791Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
184992024-02-27T21:51:12.239Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589663667Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185002024-02-27T21:51:12.239Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589697343Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185012024-02-27T21:51:12.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589896569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185022024-02-27T21:51:12.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589953563Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185032024-02-27T21:51:12.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.589988419Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185042024-02-27T21:51:12.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.59065879Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185052024-02-27T21:51:12.239Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.590695215Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185062024-02-27T21:51:12.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.590717823Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185072024-02-27T21:51:12.239Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.590749279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185082024-02-27T21:51:12.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.590770177Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185092024-02-27T21:51:12.239Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.590804593Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185102024-02-27T21:51:12.239Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.591352808Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185112024-02-27T21:51:12.239Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.591390824Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185122024-02-27T21:51:12.239Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.591408461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185132024-02-27T21:51:12.239Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.591968156Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185142024-02-27T21:51:12.239Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592035937Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185152024-02-27T21:51:12.239Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592063594Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185162024-02-27T21:51:12.239Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592076993Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185172024-02-27T21:51:12.239Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.59209734Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185182024-02-27T21:51:12.239Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592111308Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185192024-02-27T21:51:12.239Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592130066Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185202024-02-27T21:51:12.240Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592143805Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185212024-02-27T21:51:12.240Z{"msg":"[0]R flush_numbers[0..12]: [291, 310, 298, 304, 309, 307, 304, 305, 311, 310, 310, 308]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592164382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185222024-02-27T21:51:12.240Z{"msg":"[0]R generation[0..12]: [161, 170, 164, 167, 170, 170, 167, 168, 171, 170, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592179731Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185232024-02-27T21:51:12.240Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
185242024-02-27T21:51:12.240Z,"time":"2024-02-27T21:51:10.592201698Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185252024-02-27T21:51:12.240Z{"msg":"[1]R flush_numbers[0..12]: [291, 310, 298, 304, 309, 307, 304, 305, 311, 310, 310, 308]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592216246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185262024-02-27T21:51:12.240Z{"msg":"[1]R generation[0..12]: [161, 170, 164, 167, 170, 170, 167, 168, 171, 170, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592236874Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185272024-02-27T21:51:12.240Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592251612Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185282024-02-27T21:51:12.240Z{"msg":"[2]R flush_numbers[0..12]: [291, 310, 298, 304, 309, 307, 304, 305, 311, 310, 310, 308]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592277409Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185292024-02-27T21:51:12.240Z{"msg":"[2]R generation[0..12]: [161, 170, 164, 167, 170, 170, 167, 168, 171, 170, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592310895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185302024-02-27T21:51:12.240Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592346941Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185312024-02-27T21:51:12.240Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592373828Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185322024-02-27T21:51:12.240Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592388256Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185332024-02-27T21:51:12.240Z{"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592409173Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185342024-02-27T21:51:12.240Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592423132Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"":"downstairs","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185352024-02-27T21:51:12.240Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592443269Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185362024-02-27T21:51:12.240Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592455978Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"0","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185372024-02-27T21:51:12.240Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592476196Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"1","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185382024-02-27T21:51:12.240Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592489964Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"client":"2","":"downstairs","session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185392024-02-27T21:51:12.240Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592509722Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185402024-02-27T21:51:12.240Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.59252238Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185412024-02-27T21:51:12.240Z{"msg":"4eed15ac-1696-4df3-a8bc-aada3b32510e is now active with session: c0d67474-0aa4-4ab1-b230-6b2c2ad183d3","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592542768Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185422024-02-27T21:51:12.240Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
185432024-02-27T21:51:12.240Z,"time":"2024-02-27T21:51:10.592555206Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838,"session_id":"c0d67474-0aa4-4ab1-b230-6b2c2ad183d3"}
185442024-02-27T21:51:12.240Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:10.592584853Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838}
185452024-02-27T21:51:12.241ZLoading write count information from file "/tmp/test_repair_verify.data"
185462024-02-27T21:51:12.241ZRun Repair workload
185472024-02-27T21:51:12.241Z01/30 Write block 517 len 40960 data: 17 14 13 16 13 14 11 12 11 13
185482024-02-27T21:51:12.244Z02/30 Write block 367 len 32768 data: 11 11 7 8 8 6 7 9
185492024-02-27T21:51:12.244Z03/30 Read block 398 len 8192
185502024-02-27T21:51:12.244Z04/30 Read block 401 len 20480
185512024-02-27T21:51:12.244Z05/30 Write block 226 len 40960 data: 12 14 14 13 16 18 19 18 19 20
185522024-02-27T21:51:12.244Z06/30 Read block 166 len 16384
185532024-02-27T21:51:12.244Z07/30 Write block 476 len 28672 data: 12 10 10 9 11 12 15
185542024-02-27T21:51:12.245Z08/30 Read block 346 len 16384
185552024-02-27T21:51:12.245Z09/30 Read block 568 len 32768
185562024-02-27T21:51:12.245Z10/30 Read block 75 len 4096
185572024-02-27T21:51:12.245Z11/30 Read block 572 len 16384
185582024-02-27T21:51:12.248Z12/30 Read block 452 len 24576
185592024-02-27T21:51:12.248Z13/30 Read block 473 len 8192
185602024-02-27T21:51:12.248Z14/30 Read block 173 len 20480
185612024-02-27T21:51:12.248Z15/30 Write block 206 len 16384 data: 5 7 9 11
185622024-02-27T21:51:12.248Z16/30 Write block 113 len 12288 data: 16 15 13
185632024-02-27T21:51:12.248Z17/30 Read block 350 len 36864
185642024-02-27T21:51:12.251Z18/30 Read block 252 len 8192
185652024-02-27T21:51:12.251Z19/30 Write block 441 len 24576 data: 7 8 9 7 7 8
185662024-02-27T21:51:12.251Z20/30 Read block 240 len 8192
185672024-02-27T21:51:12.251Z21/30 Read block 554 len 12288
185682024-02-27T21:51:12.254Z22/30 Read block 499 len 20480
185692024-02-27T21:51:12.255Z23/30 Write block 471 len 40960 data: 11 10 11 9 11 13 11 11 10 12
185702024-02-27T21:51:12.255Z24/30 Write block 375 len 36864 data: 9 9 11 7 9 10 9 12 13
185712024-02-27T21:51:12.255Z25/30 Read block 315 len 12288
185722024-02-27T21:51:12.258Z26/30 Flush
185732024-02-27T21:51:12.262Z27/30 Write block 460 len 36864 data: 10 10 10 9 9 8 10 11 11
185742024-02-27T21:51:12.266Z28/30 Write block 170 len 12288 data: 12 11 12
185752024-02-27T21:51:12.266Z29/30 Write block 327 len 8192 data: 13 12
185762024-02-27T21:51:12.266Z30/30 Write block 43 len 40960 data: 14 15 17 14 14 13 13 14 15 16
185772024-02-27T21:51:12.266Z----------------------------------------------------------------
185782024-02-27T21:51:12.266Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
185792024-02-27T21:51:12.266ZStates: Active Active Active
185802024-02-27T21:51:12.266ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185812024-02-27T21:51:12.266Z 1 Acked 1000 Write 10 Done Done Done false
185822024-02-27T21:51:12.266Z 2 Acked 1001 Write 8 Done Done Done false
185832024-02-27T21:51:12.266Z 3 Acked 1002 Read 2 Done Done Done false
185842024-02-27T21:51:12.266Z 4 Acked 1003 Read 5 Done Sent Done false
185852024-02-27T21:51:12.266Z 5 Acked 1004 Write 10 Done Sent Done false
185862024-02-27T21:51:12.266Z 6 Acked 1005 Read 4 Done Sent Done false
185872024-02-27T21:51:12.266Z 7 Acked 1006 Write 7 Done Sent Done false
185882024-02-27T21:51:12.266Z 8 Acked 1007 Read 4 Done Sent Done false
185892024-02-27T21:51:12.266Z 9 Acked 1008 Read 8 Done Sent Done false
185902024-02-27T21:51:12.266Z 10 Acked 1009 Read 1 Done Sent Done false
185912024-02-27T21:51:12.266Z 11 Acked 1010 Read 4 Done Sent Done false
185922024-02-27T21:51:12.266Z 12 Acked 1011 Read 6 Done Sent Done false
185932024-02-27T21:51:12.266Z 13 Acked 1012 Read 2 Done Sent Done false
185942024-02-27T21:51:12.266Z 14 Acked 1013 Read 5 Done Sent Done false
185952024-02-27T21:51:12.266Z 15 Acked 1014 Write 4 Done Sent Done false
185962024-02-27T21:51:12.267Z 16 Acked 1015 Write 3 Done Sent Done false
185972024-02-27T21:51:12.267Z 17 Acked 1016 Read 9 Done Sent Done false
185982024-02-27T21:51:12.267Z 18 Acked 1017 Read 2 Done Sent Done false
185992024-02-27T21:51:12.267Z 19 Acked 1018 Write 6 Done Sent Done false
186002024-02-27T21:51:12.267Z 20 Acked 1019 Read 2 Done Sent Done false
186012024-02-27T21:51:12.267Z 21 Acked 1020 Read 3 Done Sent Done false
186022024-02-27T21:51:12.267Z 22 Acked 1021 Read 5 Done Sent Done false
186032024-02-27T21:51:12.267Z 23 Acked 1022 Write 10 Done Sent Done false
186042024-02-27T21:51:12.267Z 24 Acked 1023 Write 9 Done Sent Done false
186052024-02-27T21:51:12.267Z 25 Acked 1024 Read 3 Done Sent Done false
186062024-02-27T21:51:12.267Z 26 Acked 1025 Flush 0 Done Sent Done false
186072024-02-27T21:51:12.267Z 27 Acked 1026 Write 9 Done Sent Done false
186082024-02-27T21:51:12.267Z 28 Acked 1027 Write 3 Done Sent Done false
186092024-02-27T21:51:12.267Z 29 Acked 1028 Write 2 Done Sent Done false
186102024-02-27T21:51:12.267Z 30 Acked 1029 Write 10 Sent Sent Sent false
186112024-02-27T21:51:12.267Z STATES DS:0 DS:1 DS:2 TOTAL
186122024-02-27T21:51:12.267Z New 0 0 0 0
186132024-02-27T21:51:12.267Z Sent 1 27 1 29
186142024-02-27T21:51:12.267Z Done 29 3 29 61
186152024-02-27T21:51:12.267Z Skipped 0 0 0 0
186162024-02-27T21:51:12.267Z Error 0 0 0 0
186172024-02-27T21:51:12.267ZLast Flush: 1025 0 1025
186182024-02-27T21:51:12.267ZDownstairs last five completed:
186192024-02-27T21:51:12.268ZUpstairs last five completed: 30 29 28 27 26
186202024-02-27T21:51:12.268Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:10.624339752Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1838}
186212024-02-27T21:51:12.268ZWrote out file "/tmp/test_repair_verify.data"
186222024-02-27T21:51:12.274Z
186232024-02-27T21:51:12.277ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
186242024-02-27T21:51:12.289ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186252024-02-27T21:51:12.289Z 0 000-019 161 161 161 291 291 291 F F F
186262024-02-27T21:51:12.289Z 1 020-039 170 170 170 310 310 310 F F F
186272024-02-27T21:51:12.289Z 2 040-059 164 164 164 298 298 298 T F T <---
186282024-02-27T21:51:12.289Z 3 060-079 167 167 167 304 304 304 F F F
186292024-02-27T21:51:12.289Z 4 080-099 170 170 170 309 309 309 F F F
186302024-02-27T21:51:12.292Z 5 100-119 172 170 172 312 307 312 F F F <---
186312024-02-27T21:51:12.292Z 6 120-139 167 167 167 304 304 304 F F F
186322024-02-27T21:51:12.292Z 7 140-159 168 168 168 305 305 305 F F F
186332024-02-27T21:51:12.292Z 8 160-179 171 171 171 311 311 311 T F T <---
186342024-02-27T21:51:12.292Z 9 180-199 170 170 170 310 310 310 F F F
186352024-02-27T21:51:12.292Z 10 200-219 172 170 172 312 310 312 F F F <---
186362024-02-27T21:51:12.292Z 11 220-239 172 170 172 312 308 312 F F F <---
186372024-02-27T21:51:12.292Z 12 240-259 166 166 166 302 302 302 F F F
186382024-02-27T21:51:12.293Z 13 260-279 170 170 170 310 310 310 F F F
186392024-02-27T21:51:12.293Z 14 280-299 170 170 170 308 308 308 F F F
186402024-02-27T21:51:12.293Z 15 300-319 170 170 170 310 310 310 F F F
186412024-02-27T21:51:12.293Z 16 320-339 166 166 166 302 302 302 T F T <---
186422024-02-27T21:51:12.293Z 17 340-359 164 164 164 299 299 299 F F F
186432024-02-27T21:51:12.293Z 18 360-379 172 170 172 312 308 312 F T F <---
186442024-02-27T21:51:12.293Z 19 380-399 172 170 172 312 308 312 F F F <---
186452024-02-27T21:51:12.293Z 20 400-419 168 168 168 305 305 305 F F F
186462024-02-27T21:51:12.293Z 21 420-439 171 171 171 311 311 311 F F F
186472024-02-27T21:51:12.293Z 22 440-459 172 168 172 312 305 312 F F F <---
186482024-02-27T21:51:12.293Z 23 460-479 172 160 172 312 289 312 T F T <---
186492024-02-27T21:51:12.293Z 24 480-499 172 168 172 312 305 312 F F F <---
186502024-02-27T21:51:12.293Z 25 500-519 172 170 172 312 307 312 F T F <---
186512024-02-27T21:51:12.293Z 26 520-539 172 170 172 312 307 312 F T F <---
186522024-02-27T21:51:12.293Z 27 540-559 166 166 166 303 303 303 F F F
186532024-02-27T21:51:12.293Z 28 560-579 164 164 164 299 299 299 F F F
186542024-02-27T21:51:12.293Z 29 580-599 168 168 168 305 305 305 F F F
186552024-02-27T21:51:12.293ZMax gen: 172, Max flush: 312
186562024-02-27T21:51:12.293ZError: Difference in extent metadata found!
186572024-02-27T21:51:12.293ZA Difference in extent metadata is expected here
186582024-02-27T21:51:12.293ZOn loop 86
186592024-02-27T21:51:12.293Z
186602024-02-27T21:51:12.296ZVerifying data now
186612024-02-27T21:51:17.974ZNo change after verify
186622024-02-27T21:51:17.974ZLoop: 86 Downstairs dump after verify (and repair):
186632024-02-27T21:51:17.982ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186642024-02-27T21:51:17.982Z 0 000-019 161 161 161 291 291 291 F F F
186652024-02-27T21:51:17.982Z 1 020-039 170 170 170 310 310 310 F F F
186662024-02-27T21:51:17.983Z 2 040-059 173 173 173 313 313 313 F F F
186672024-02-27T21:51:17.983Z 3 060-079 167 167 167 304 304 304 F F F
186682024-02-27T21:51:17.983Z 4 080-099 170 170 170 309 309 309 F F F
186692024-02-27T21:51:17.984Z 5 100-119 172 172 172 312 312 312 F F F
186702024-02-27T21:51:17.984Z 6 120-139 167 167 167 304 304 304 F F F
186712024-02-27T21:51:17.984Z 7 140-159 168 168 168 305 305 305 F F F
186722024-02-27T21:51:17.984Z 8 160-179 173 173 173 313 313 313 F F F
186732024-02-27T21:51:17.984Z 9 180-199 170 170 170 310 310 310 F F F
186742024-02-27T21:51:17.985Z 10 200-219 172 172 172 312 312 312 F F F
186752024-02-27T21:51:17.985Z 11 220-239 172 172 172 312 312 312 F F F
186762024-02-27T21:51:17.985Z 12 240-259 166 166 166 302 302 302 F F F
186772024-02-27T21:51:17.985Z 13 260-279 170 170 170 310 310 310 F F F
186782024-02-27T21:51:17.986Z 14 280-299 170 170 170 308 308 308 F F F
186792024-02-27T21:51:17.986Z 15 300-319 170 170 170 310 310 310 F F F
186802024-02-27T21:51:17.986Z 16 320-339 173 173 173 313 313 313 F F F
186812024-02-27T21:51:17.986Z 17 340-359 164 164 164 299 299 299 F F F
186822024-02-27T21:51:17.987Z 18 360-379 172 172 172 312 312 312 F F F
186832024-02-27T21:51:17.987Z 19 380-399 172 172 172 312 312 312 F F F
186842024-02-27T21:51:17.987Z 20 400-419 168 168 168 305 305 305 F F F
186852024-02-27T21:51:17.987Z 21 420-439 171 171 171 311 311 311 F F F
186862024-02-27T21:51:17.988Z 22 440-459 172 172 172 312 312 312 F F F
186872024-02-27T21:51:17.988Z 23 460-479 173 173 173 313 313 313 F F F
186882024-02-27T21:51:17.988Z 24 480-499 172 172 172 312 312 312 F F F
186892024-02-27T21:51:17.988Z 25 500-519 172 172 172 312 312 312 F F F
186902024-02-27T21:51:17.988Z 26 520-539 172 172 172 312 312 312 F F F
186912024-02-27T21:51:17.988Z 27 540-559 166 166 166 303 303 303 F F F
186922024-02-27T21:51:17.989Z 28 560-579 164 164 164 299 299 299 F F F
186932024-02-27T21:51:17.989Z 29 580-599 168 168 168 305 305 305 F F F
186942024-02-27T21:51:17.989ZMax gen: 173, Max flush: 313
186952024-02-27T21:51:17.989Z
186962024-02-27T21:51:17.990ZBegin loop 87
186972024-02-27T21:51:17.990ZDownstairs to restart: 0
186982024-02-27T21:51:18.532Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339206206Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846}
186992024-02-27T21:51:18.532Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339452717Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846}
187002024-02-27T21:51:18.533Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339470286Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846}
187012024-02-27T21:51:18.533Z{"msg":"Crucible fd8a73d4-e780-4140-89dd-de1b515369d1 has session id: f6a99f9e-361b-452f-8b01-c803778d569d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339491183Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187022024-02-27T21:51:18.533Z{"msg":"Upstairs opts: Upstairs UUID: fd8a73d4-e780-4140-89dd-de1b515369d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339534528Z","hostname":"Crucible runtime is spawned
187032024-02-27T21:51:18.533Zip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187042024-02-27T21:51:18.533Z{"msg":"Crucible stats registered with UUID: fd8a73d4-e780-4140-89dd-de1b515369d1","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339577213Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187052024-02-27T21:51:18.534Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339648425Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846}
187062024-02-27T21:51:18.534Z{"msg":"fd8a73d4-e780-4140-89dd-de1b515369d1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.33977403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187072024-02-27T21:51:18.534Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339811335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187082024-02-27T21:51:18.534Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339866109Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187092024-02-27T21:51:18.534Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339901915Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187102024-02-27T21:51:18.534Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.33993966Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187112024-02-27T21:51:18.534Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.339978316Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187122024-02-27T21:51:18.534Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340013211Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187132024-02-27T21:51:18.535Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340064125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187142024-02-27T21:51:18.535Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340116259Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187152024-02-27T21:51:18.535Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340165853Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187162024-02-27T21:51:18.535Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340214218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187172024-02-27T21:51:18.535Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340248684Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187182024-02-27T21:51:18.535Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340340843Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187192024-02-27T21:51:18.535Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340880659Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187202024-02-27T21:51:18.535Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340934973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187212024-02-27T21:51:18.536Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.34096035Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187222024-02-27T21:51:18.536Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.340990616Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187232024-02-27T21:51:18.536Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.341477999Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187242024-02-27T21:51:18.536Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.341528433Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187252024-02-27T21:51:18.536Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.341690284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187262024-02-27T21:51:18.536Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.341733429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187272024-02-27T21:51:18.536Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342093137Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187282024-02-27T21:51:18.537Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342130892Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187292024-02-27T21:51:18.537Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.34214744Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187302024-02-27T21:51:18.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342168188Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187312024-02-27T21:51:18.537Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342182446Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187322024-02-27T21:51:18.537Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342203024Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187332024-02-27T21:51:18.537Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342392041Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187342024-02-27T21:51:18.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342801153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187352024-02-27T21:51:18.537Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342840789Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187362024-02-27T21:51:18.538Z{"msg":"[0]R flush_numbers[0..12]: [291, 310, 313, 304, 309, 312, 304, 305, 313, 310, 312, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342868285Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187372024-02-27T21:51:18.538Z{"msg":"[0]R generation[0..12]: [161, 170, 173, 167, 170, 172, 167, 168, 173, 170, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342897382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187382024-02-27T21:51:18.538Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342923569Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187392024-02-27T21:51:18.538Z{"msg":"[1]R flush_numbers[0..12]: [291, 310, 313, 304, 309, 312, 304, 305, 313, 310, 312, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.342951815Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187402024-02-27T21:51:18.538Z{"msg":"[1]R generation[0..12]: [161, 170, 173, 167, 170, 172, 167, 168, 173, 170, 172, 172]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
187412024-02-27T21:51:18.538Zcrucible","level":30,"time":"2024-02-27T21:51:16.342988771Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187422024-02-27T21:51:18.538Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343038545Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187432024-02-27T21:51:18.538Z{"msg":"[2]R flush_numbers[0..12]: [291, 310, 313, 304, 309, 312, 304, 305, 313, 310, 312, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343073821Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187442024-02-27T21:51:18.538Z{"msg":"[2]R generation[0..12]: [161, 170, 173, 167, 170, 172, 167, 168, 173, 170, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343099838Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187452024-02-27T21:51:18.538Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343132934Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187462024-02-27T21:51:18.539Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343156991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187472024-02-27T21:51:18.539Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343185978Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187482024-02-27T21:51:18.539Z{"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343209205Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187492024-02-27T21:51:18.539Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343235782Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"":"downstairs","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187502024-02-27T21:51:18.539Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343259419Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187512024-02-27T21:51:18.540Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343284466Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"0","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187522024-02-27T21:51:18.540Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187532024-02-27T21:51:18.540Z,"time":"2024-02-27T21:51:16.343308083Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"1","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187542024-02-27T21:51:18.540Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343345919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"client":"2","":"downstairs","session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187552024-02-27T21:51:18.540Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343377575Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187562024-02-27T21:51:18.540Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343400483Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187572024-02-27T21:51:18.540Z{"msg":"fd8a73d4-e780-4140-89dd-de1b515369d1 is now active with session: f6a99f9e-361b-452f-8b01-c803778d569d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343426909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187582024-02-27T21:51:18.540Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343448287Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846,"session_id":"f6a99f9e-361b-452f-8b01-c803778d569d"}
187592024-02-27T21:51:18.541Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:16.343473894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846}
187602024-02-27T21:51:18.541ZLoading write count information from file "/tmp/test_repair_verify.data"
187612024-02-27T21:51:18.541ZRun Repair workload
187622024-02-27T21:51:18.541Z01/30 Write block 179 len 4096 data: 13
187632024-02-27T21:51:18.541Z02/30 Read block 554 len 32768
187642024-02-27T21:51:18.541Z03/30 Write block 346 len 28672 data: 16 16 20 19 18 17 14
187652024-02-27T21:51:18.541Z04/30 Read block 582 len 24576
187662024-02-27T21:51:18.542Z05/30 Read block 376 len 4096
187672024-02-27T21:51:18.542Z06/30 Write block 591 len 24576 data: 11 13 14 11 10 11
187682024-02-27T21:51:18.542Z07/30 Read block 172 len 16384
187692024-02-27T21:51:18.542Z08/30 Write block 490 len 24576 data: 14 14 12 9 12 10
187702024-02-27T21:51:18.542Z09/30 Write block 400 len 28672 data: 16 15 13 13 13 11 11
187712024-02-27T21:51:18.542Z10/30 Write block 521 len 12288 data: 14 15 12
187722024-02-27T21:51:18.542Z11/30 Write block 11 len 4096 data: 17
187732024-02-27T21:51:18.542Z12/30 Flush
187742024-02-27T21:51:18.562Z13/30 Write block 335 len 28672 data: 19 15 15 17 17 20 20
187752024-02-27T21:51:18.563Z14/30 Read block 315 len 12288
187762024-02-27T21:51:18.563Z15/30 Read block 438 len 36864
187772024-02-27T21:51:18.563Z16/30 Write block 379 len 20480 data: 10 11 10 13 14
187782024-02-27T21:51:18.563Z17/30 Read block 141 len 8192
187792024-02-27T21:51:18.563Z18/30 Read block 523 len 24576
187802024-02-27T21:51:18.563Z19/30 Write block 289 len 28672 data: 15 15 17 16 20 19 20
187812024-02-27T21:51:18.563Z20/30 Read block 549 len 4096
187822024-02-27T21:51:18.563Z21/30 Read block 526 len 16384
187832024-02-27T21:51:18.564Z22/30 Read block 142 len 24576
187842024-02-27T21:51:18.564Z23/30 Read block 434 len 8192
187852024-02-27T21:51:18.564Z24/30 Write block 568 len 12288 data: 11 11 9
187862024-02-27T21:51:18.564Z25/30 Read block 336 len 28672
187872024-02-27T21:51:18.564Z26/30 Read block 65 len 36864
187882024-02-27T21:51:18.564Z27/30 Read block 439 len 36864
187892024-02-27T21:51:18.564Z28/30 Write block 527 len 12288 data: 14 12 11
187902024-02-27T21:51:18.564Z29/30 Read block 575 len 12288
187912024-02-27T21:51:18.565Z30/30 Write block 465 len 24576 data: 9 11 12 12 13 10
187922024-02-27T21:51:18.565Z----------------------------------------------------------------
187932024-02-27T21:51:18.565Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
187942024-02-27T21:51:18.565ZStates: Active Active Active
187952024-02-27T21:51:18.565ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187962024-02-27T21:51:18.565Z 1 Acked 1000 Write 1 Done Done Done false
187972024-02-27T21:51:18.565Z 2 Acked 1001 Read 8 Done Done Done false
187982024-02-27T21:51:18.565Z 3 Acked 1002 Write 7 Done Done Done false
187992024-02-27T21:51:18.566Z 4 Acked 1003 Read 6 Done Done Done false
188002024-02-27T21:51:18.566Z 5 Acked 1004 Read 1 Sent Done Done false
188012024-02-27T21:51:18.566Z 6 Acked 1005 Write 6 Sent Done Done false
188022024-02-27T21:51:18.566Z 7 Acked 1006 Read 4 Sent Done Done false
188032024-02-27T21:51:18.566Z 8 Acked 1007 Write 6 Sent Done Done false
188042024-02-27T21:51:18.566Z 9 Acked 1008 Write 7 Sent Done Done false
188052024-02-27T21:51:18.566Z 10 Acked 1009 Write 3 Sent Done Done false
188062024-02-27T21:51:18.566Z 11 Acked 1010 Write 1 Sent Done Done false
188072024-02-27T21:51:18.566Z 12 Acked 1011 Flush 0 Sent Done Done false
188082024-02-27T21:51:18.567Z 13 Acked 1012 Write 7 Sent Done Done false
188092024-02-27T21:51:18.567Z 14 Acked 1013 Read 3 Sent Done Done false
188102024-02-27T21:51:18.567Z 15 Acked 1014 Read 9 Sent Done Done false
188112024-02-27T21:51:18.567Z 16 Acked 1015 Write 5 Sent Done Done false
188122024-02-27T21:51:18.567Z 17 Acked 1016 Read 2 Sent Done Done false
188132024-02-27T21:51:18.567Z 18 Acked 1017 Read 6 Sent Done Done false
188142024-02-27T21:51:18.567Z 19 Acked 1018 Write 7 Sent Done Done false
188152024-02-27T21:51:18.567Z 20 Acked 1019 Read 1 Sent Done Done false
188162024-02-27T21:51:18.567Z 21 Acked 1020 Read 4 Sent Done Done false
188172024-02-27T21:51:18.568Z 22 Acked 1021 Read 6 Sent Done Done false
188182024-02-27T21:51:18.568Z 23 Acked 1022 Read 2 Sent Done Done false
188192024-02-27T21:51:18.568Z 24 Acked 1023 Write 3 Sent Done Done false
188202024-02-27T21:51:18.568Z 25 Acked 1024 Read 7 Sent Done Done false
188212024-02-27T21:51:18.568Z 26 Acked 1025 Read 9 Sent Done Done false
188222024-02-27T21:51:18.568Z 27 Acked 1026 Read 9 Sent Done Done false
188232024-02-27T21:51:18.568Z 28 Acked 1027 Write 3 Sent Done Done false
188242024-02-27T21:51:18.568Z 29 Acked 1028 Read 3 Sent Done Done false
188252024-02-27T21:51:18.568Z 30 Acked 1029 Write 6 Sent Sent Sent false
188262024-02-27T21:51:18.569Z STATES DS:0 DS:1 DS:2 TOTAL
188272024-02-27T21:51:18.569Z New 0 0 0 0
188282024-02-27T21:51:18.569Z Sent 26 1 1 28
188292024-02-27T21:51:18.569Z Done 4 29 29 62
188302024-02-27T21:51:18.569Z Skipped 0 0 0 0
188312024-02-27T21:51:18.569Z Error 0 0 0 0
188322024-02-27T21:51:18.569ZLast Flush: 0 1011 1011
188332024-02-27T21:51:18.570ZDownstairs last five completed:
188342024-02-27T21:51:18.570ZUpstairs last five completed: 30 29 28 27 26
188352024-02-27T21:51:18.570Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:16.372322686Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1846}
188362024-02-27T21:51:18.570ZWrote out file "/tmp/test_repair_verify.data"
188372024-02-27T21:51:18.570Z
188382024-02-27T21:51:18.570ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
188392024-02-27T21:51:18.570ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188402024-02-27T21:51:18.571Z 0 000-019 161 174 174 291 314 314 F F F <---
188412024-02-27T21:51:18.571Z 1 020-039 170 170 170 310 310 310 F F F
188422024-02-27T21:51:18.571Z 2 040-059 173 173 173 313 313 313 F F F
188432024-02-27T21:51:18.571Z 3 060-079 167 167 167 304 304 304 F F F
188442024-02-27T21:51:18.571Z 4 080-099 170 170 170 309 309 309 F F F
188452024-02-27T21:51:18.571Z 5 100-119 172 172 172 312 312 312 F F F
188462024-02-27T21:51:18.571Z 6 120-139 167 167 167 304 304 304 F F F
188472024-02-27T21:51:18.571Z 7 140-159 168 168 168 305 305 305 F F F
188482024-02-27T21:51:18.571Z 8 160-179 173 174 174 313 314 314 T F F <---
188492024-02-27T21:51:18.572Z 9 180-199 170 170 170 310 310 310 F F F
188502024-02-27T21:51:18.572Z 10 200-219 172 172 172 312 312 312 F F F
188512024-02-27T21:51:18.572Z 11 220-239 172 172 172 312 312 312 F F F
188522024-02-27T21:51:18.572Z 12 240-259 166 166 166 302 302 302 F F F
188532024-02-27T21:51:18.572Z 13 260-279 170 170 170 310 310 310 F F F
188542024-02-27T21:51:18.572Z 14 280-299 170 170 170 308 308 308 F T T <---
188552024-02-27T21:51:18.572Z 15 300-319 170 170 170 310 310 310 F F F
188562024-02-27T21:51:18.572Z 16 320-339 173 173 173 313 313 313 F T T <---
188572024-02-27T21:51:18.572Z 17 340-359 164 174 174 299 314 314 T T T <---
188582024-02-27T21:51:18.573Z 18 360-379 172 172 172 312 312 312 F T T <---
188592024-02-27T21:51:18.573Z 19 380-399 172 172 172 312 312 312 F T T <---
188602024-02-27T21:51:18.573Z 20 400-419 168 174 174 305 314 314 F F F <---
188612024-02-27T21:51:18.573Z 21 420-439 171 171 171 311 311 311 F F F
188622024-02-27T21:51:18.573Z 22 440-459 172 172 172 312 312 312 F F F
188632024-02-27T21:51:18.573Z 23 460-479 173 173 173 313 313 313 F T T <---
188642024-02-27T21:51:18.573Z 24 480-499 172 174 174 312 314 314 F F F <---
188652024-02-27T21:51:18.573Z 25 500-519 172 172 172 312 312 312 F F F
188662024-02-27T21:51:18.573Z 26 520-539 172 174 174 312 314 314 F T T <---
188672024-02-27T21:51:18.573Z 27 540-559 166 166 166 303 303 303 F F F
188682024-02-27T21:51:18.574Z 28 560-579 164 164 164 299 299 299 F T T <---
188692024-02-27T21:51:18.574Z 29 580-599 168 174 174 305 314 314 F F F <---
188702024-02-27T21:51:18.574ZMax gen: 174, Max flush: 314
188712024-02-27T21:51:18.574ZError: Difference in extent metadata found!
188722024-02-27T21:51:18.574ZA Difference in extent metadata is expected here
188732024-02-27T21:51:18.574ZOn loop 87
188742024-02-27T21:51:18.579Z
188752024-02-27T21:51:18.580ZVerifying data now
188762024-02-27T21:51:23.717ZNo change after verify
188772024-02-27T21:51:23.717ZLoop: 87 Downstairs dump after verify (and repair):
188782024-02-27T21:51:23.739ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188792024-02-27T21:51:23.739Z 0 000-019 174 174 174 314 314 314 F F F
188802024-02-27T21:51:23.740Z 1 020-039 170 170 170 310 310 310 F F F
188812024-02-27T21:51:23.740Z 2 040-059 173 173 173 313 313 313 F F F
188822024-02-27T21:51:23.740Z 3 060-079 167 167 167 304 304 304 F F F
188832024-02-27T21:51:23.761Z 4 080-099 170 170 170 309 309 309 F F F
188842024-02-27T21:51:23.761Z 5 100-119 172 172 172 312 312 312 F F F
188852024-02-27T21:51:23.762Z 6 120-139 167 167 167 304 304 304 F F F
188862024-02-27T21:51:23.762Z 7 140-159 168 168 168 305 305 305 F F F
188872024-02-27T21:51:23.762Z 8 160-179 174 174 174 314 314 314 F F F
188882024-02-27T21:51:23.763Z 9 180-199 170 170 170 310 310 310 F F F
188892024-02-27T21:51:23.763Z 10 200-219 172 172 172 312 312 312 F F F
188902024-02-27T21:51:23.763Z 11 220-239 172 172 172 312 312 312 F F F
188912024-02-27T21:51:23.763Z 12 240-259 166 166 166 302 302 302 F F F
188922024-02-27T21:51:23.763Z 13 260-279 170 170 170 310 310 310 F F F
188932024-02-27T21:51:23.763Z 14 280-299 175 175 175 315 315 315 F F F
188942024-02-27T21:51:23.763Z 15 300-319 170 170 170 310 310 310 F F F
188952024-02-27T21:51:23.764Z 16 320-339 175 175 175 315 315 315 F F F
188962024-02-27T21:51:23.764Z 17 340-359 175 175 175 315 315 315 F F F
188972024-02-27T21:51:23.764Z 18 360-379 175 175 175 315 315 315 F F F
188982024-02-27T21:51:23.764Z 19 380-399 175 175 175 315 315 315 F F F
188992024-02-27T21:51:23.764Z 20 400-419 174 174 174 314 314 314 F F F
189002024-02-27T21:51:23.764Z 21 420-439 171 171 171 311 311 311 F F F
189012024-02-27T21:51:23.765Z 22 440-459 172 172 172 312 312 312 F F F
189022024-02-27T21:51:23.765Z 23 460-479 175 175 175 315 315 315 F F F
189032024-02-27T21:51:23.766Z 24 480-499 174 174 174 314 314 314 F F F
189042024-02-27T21:51:23.766Z 25 500-519 172 172 172 312 312 312 F F F
189052024-02-27T21:51:23.766Z 26 520-539 175 175 175 315 315 315 F F F
189062024-02-27T21:51:23.766Z 27 540-559 166 166 166 303 303 303 F F F
189072024-02-27T21:51:23.766Z 28 560-579 175 175 175 315 315 315 F F F
189082024-02-27T21:51:23.767Z 29 580-599 174 174 174 314 314 314 F F F
189092024-02-27T21:51:23.767ZMax gen: 175, Max flush: 315
189102024-02-27T21:51:23.767Z
189112024-02-27T21:51:23.767ZBegin loop 88
189122024-02-27T21:51:23.767ZDownstairs to restart: 0
189132024-02-27T21:51:23.789Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107101093Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854}
189142024-02-27T21:51:23.790Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107402517Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854}
189152024-02-27T21:51:23.790ZCrucible runtime is spawned
189162024-02-27T21:51:23.790Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107453861Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854}
189172024-02-27T21:51:23.790Z{"msg":"Crucible 7ba4f547-6352-4865-9df8-329de4eab64c has session id: b5f08026-3d8e-47a4-9c19-441459b8f72d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107483918Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189182024-02-27T21:51:23.790Z{"msg":"Upstairs opts: Upstairs UUID: 7ba4f547-6352-4865-9df8-329de4eab64c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107526083Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189192024-02-27T21:51:23.791Z{"msg":"Crucible stats registered with UUID: 7ba4f547-6352-4865-9df8-329de4eab64c","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107567298Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189202024-02-27T21:51:23.791Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107596494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854}
189212024-02-27T21:51:23.791Z{"msg":"7ba4f547-6352-4865-9df8-329de4eab64c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107628481Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189222024-02-27T21:51:23.791Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107656467Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189232024-02-27T21:51:23.792Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107697882Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189242024-02-27T21:51:23.792Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107729619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189252024-02-27T21:51:23.792Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107771774Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189262024-02-27T21:51:23.792Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107813759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189272024-02-27T21:51:23.793Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107861983Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189282024-02-27T21:51:23.793Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.107907888Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189292024-02-27T21:51:23.794Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108035003Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189302024-02-27T21:51:23.794Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108064879Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189312024-02-27T21:51:23.794Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108101175Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189322024-02-27T21:51:23.794Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108133741Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189332024-02-27T21:51:23.795Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108158158Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189342024-02-27T21:51:23.795Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108713213Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189352024-02-27T21:51:23.795Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108772246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189362024-02-27T21:51:23.795Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.10882226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189372024-02-27T21:51:23.796Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108855016Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189382024-02-27T21:51:23.796Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108882883Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189392024-02-27T21:51:23.796Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.108913239Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189402024-02-27T21:51:23.796Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.109368696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189412024-02-27T21:51:23.796Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.109407011Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189422024-02-27T21:51:23.796Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.109777297Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189432024-02-27T21:51:23.797Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.109886195Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189442024-02-27T21:51:23.797Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.10992302Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189452024-02-27T21:51:23.797Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.109939188Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189462024-02-27T21:51:23.797Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.109960226Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189472024-02-27T21:51:23.797Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.109974234Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189482024-02-27T21:51:23.798Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.109994122Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189492024-02-27T21:51:23.798Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110211936Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189502024-02-27T21:51:23.798Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110250622Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189512024-02-27T21:51:23.798Z{"msg":"[0]R flush_numbers[0..12]: [314, 310, 313, 304, 309, 312, 304, 305, 314, 310, 312, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110286268Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189522024-02-27T21:51:23.798Z{"msg":"[0]R generation[0..12]: [174, 170, 173, 167, 170, 172, 167, 168, 174, 170, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110304505Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189532024-02-27T21:51:23.798Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110326393Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189542024-02-27T21:51:23.798Z{"msg":"[1]R flush_numbers[0..12]: [314, 310, 313, 304, 309, 312, 304, 305, 314, 310, 312, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110341281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189552024-02-27T21:51:23.799Z{"msg":"[1]R generation[0..12]: [174, 170, 173, 167, 170, 172, 167, 168, 174, 170, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110361619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189562024-02-27T21:51:23.799Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110379977Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189572024-02-27T21:51:23.799Z{"msg":"Wait for a query_work_queue command to finish before sending IO
189582024-02-27T21:51:23.799Z[2]R flush_numbers[0..12]: [314, 310, 313, 304, 309, 312, 304, 305, 314, 310, 312, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110417392Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189592024-02-27T21:51:23.799Z{"msg":"[2]R generation[0..12]: [174, 170, 173, 167, 170, 172, 167, 168, 174, 170, 172, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110455768Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189602024-02-27T21:51:23.799Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110480235Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189612024-02-27T21:51:23.799Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110494193Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189622024-02-27T21:51:23.800Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.11052002Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189632024-02-27T21:51:23.800Z{"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110555186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189642024-02-27T21:51:23.800Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110588452Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"":"downstairs","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189652024-02-27T21:51:23.800Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.11060566Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189662024-02-27T21:51:23.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110629967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"0","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189672024-02-27T21:51:23.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110665173Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"1","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189682024-02-27T21:51:23.800Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110698119Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"client":"2","":"downstairs","session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189692024-02-27T21:51:23.800Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110714147Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189702024-02-27T21:51:23.800Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110740674Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189712024-02-27T21:51:23.801Z{"msg":"7ba4f547-6352-4865-9df8-329de4eab64c is now active with session: b5f08026-3d8e-47a4-9c19-441459b8f72d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.11077749Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189722024-02-27T21:51:23.801Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110804336Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854,"session_id":"b5f08026-3d8e-47a4-9c19-441459b8f72d"}
189732024-02-27T21:51:23.801Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:22.110818785Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854}
189742024-02-27T21:51:23.801ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189752024-02-27T21:51:23.801ZLoading write count information from file "/tmp/test_repair_verify.data"
189762024-02-27T21:51:23.801ZRun Repair workload
189772024-02-27T21:51:23.801Z01/30 Write block 207 len 12288 data: 8 10 12
189782024-02-27T21:51:23.801Z02/30 Flush
189792024-02-27T21:51:23.802Z03/30 Write block 531 len 40960 data: 9 7 8 11 11 10 10 9 9 9
189802024-02-27T21:51:23.802Z04/30 Flush
189812024-02-27T21:51:23.802Z05/30 Write block 328 len 40960 data: 13 13 13 13 13 15 16 20 16 16
189822024-02-27T21:51:23.802Z06/30 Read block 156 len 20480
189832024-02-27T21:51:23.802Z07/30 Write block 137 len 12288 data: 11 11 13
189842024-02-27T21:51:23.802Z08/30 Write block 502 len 28672 data: 8 8 9 10 11 10 7
189852024-02-27T21:51:23.802Z09/30 Write block 513 len 20480 data: 17 16 13 14 18
189862024-02-27T21:51:23.802Z10/30 Read block 15 len 16384
189872024-02-27T21:51:23.802Z11/30 Flush
189882024-02-27T21:51:23.803Z12/30 Flush
189892024-02-27T21:51:23.803Z13/30 Write block 167 len 20480 data: 10 9 11 13 12
189902024-02-27T21:51:23.803Z14/30 Write block 108 len 16384 data: 18 16 15 19
189912024-02-27T21:51:23.803Z15/30 Read block 182 len 16384
189922024-02-27T21:51:23.803Z16/30 Read block 268 len 36864
189932024-02-27T21:51:23.803Z17/30 Flush
189942024-02-27T21:51:23.821Z18/30 Write block 108 len 4096 data: 19
189952024-02-27T21:51:23.821Z19/30 Write block 562 len 24576 data: 17 15 14 11 9 9
189962024-02-27T21:51:23.821Z20/30 Write block 17 len 4096 data: 17
189972024-02-27T21:51:23.821Z21/30 Write block 461 len 20480 data: 11 11 10 10 10
189982024-02-27T21:51:23.821Z22/30 Write block 140 len 28672 data: 13 13 15 13 11 10 11
189992024-02-27T21:51:23.822Z23/30 Write block 478 len 40960 data: 12 11 13 13 16 16 14 14 17 18
190002024-02-27T21:51:23.822Z24/30 Read block 572 len 8192
190012024-02-27T21:51:23.822Z25/30 Write block 129 len 4096 data: 12
190022024-02-27T21:51:23.822Z26/30 Read block 56 len 8192
190032024-02-27T21:51:23.822Z27/30 Write block 387 len 8192 data: 13 11
190042024-02-27T21:51:23.822Z28/30 Write block 171 len 36864 data: 13 13 11 15 13 15 13 15 14
190052024-02-27T21:51:23.822Z29/30 Write block 465 len 16384 data: 11 12 13 13
190062024-02-27T21:51:23.822Z30/30 Write block 394 len 4096 data: 12
190072024-02-27T21:51:23.823Z----------------------------------------------------------------
190082024-02-27T21:51:23.823Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
190092024-02-27T21:51:23.823ZStates: Active Active Active
190102024-02-27T21:51:23.823ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
190112024-02-27T21:51:23.823Z 1 Acked 1000 Write 3 Done Done Done false
190122024-02-27T21:51:23.823Z 2 Acked 1001 Flush 0 Sent Done Done false
190132024-02-27T21:51:23.824Z 3 Acked 1002 Write 10 Sent Done Done false
190142024-02-27T21:51:23.824Z 4 Acked 1003 Flush 0 Sent Done Done false
190152024-02-27T21:51:23.824Z 5 Acked 1004 Write 10 Sent Done Done false
190162024-02-27T21:51:23.824Z 6 Acked 1005 Read 5 Sent Done Done false
190172024-02-27T21:51:23.824Z 7 Acked 1006 Write 3 Sent Done Done false
190182024-02-27T21:51:23.824Z 8 Acked 1007 Write 7 Sent Done Done false
190192024-02-27T21:51:23.824Z 9 Acked 1008 Write 5 Sent Done Done false
190202024-02-27T21:51:23.825Z 10 Acked 1009 Read 4 Sent Done Done false
190212024-02-27T21:51:23.825Z 11 Acked 1010 Flush 0 Sent Done Done false
190222024-02-27T21:51:23.825Z 12 Acked 1011 Flush 0 Sent Done Done false
190232024-02-27T21:51:23.825Z 13 Acked 1012 Write 5 Sent Done Done false
190242024-02-27T21:51:23.825Z 14 Acked 1013 Write 4 Sent Done Done false
190252024-02-27T21:51:23.826Z 15 Acked 1014 Read 4 Sent Done Done false
190262024-02-27T21:51:23.826Z 16 Acked 1015 Read 9 Sent Done Done false
190272024-02-27T21:51:23.826Z 17 Acked 1016 Flush 0 Sent Done Done false
190282024-02-27T21:51:23.826Z 18 Acked 1017 Write 1 Sent Done Done false
190292024-02-27T21:51:23.826Z 19 Acked 1018 Write 6 Sent Done Done false
190302024-02-27T21:51:23.826Z 20 Acked 1019 Write 1 Sent Done Done false
190312024-02-27T21:51:23.826Z 21 Acked 1020 Write 5 Sent Done Done false
190322024-02-27T21:51:23.826Z 22 Acked 1021 Write 7 Sent Done Done false
190332024-02-27T21:51:23.826Z 23 Acked 1022 Write 10 Sent Done Done false
190342024-02-27T21:51:23.826Z 24 Acked 1023 Read 2 Sent Done Done false
190352024-02-27T21:51:23.827Z 25 Acked 1024 Write 1 Sent Done Done false
190362024-02-27T21:51:23.827Z 26 Acked 1025 Read 2 Sent Done Done false
190372024-02-27T21:51:23.827Z 27 Acked 1026 Write 2 Sent Done Done false
190382024-02-27T21:51:23.827Z 28 Acked 1027 Write 9 Sent Done Done false
190392024-02-27T21:51:23.827Z 29 Acked 1028 Write 4 Sent Sent Sent false
190402024-02-27T21:51:23.827Z 30 Acked 1029 Write 1 Sent Sent Sent false
190412024-02-27T21:51:23.827Z STATES DS:0 DS:1 DS:2 TOTAL
190422024-02-27T21:51:23.828Z New 0 0 0 0
190432024-02-27T21:51:23.828Z Sent 29 2 2 33
190442024-02-27T21:51:23.828Z Done 1 28 28 57
190452024-02-27T21:51:23.828Z Skipped 0 0 0 0
190462024-02-27T21:51:23.828Z Error 0 0 0 0
190472024-02-27T21:51:23.828ZLast Flush: 0 1016 1016
190482024-02-27T21:51:23.828ZDownstairs last five completed:
190492024-02-27T21:51:23.829ZUpstairs last five completed: 30 29 28 27 26
190502024-02-27T21:51:23.829Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:22.142030288Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1854}
190512024-02-27T21:51:23.829ZWrote out file "/tmp/test_repair_verify.data"
190522024-02-27T21:51:23.829Z
190532024-02-27T21:51:23.829ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190542024-02-27T21:51:23.829ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190552024-02-27T21:51:23.829Z 0 000-019 174 174 174 314 314 314 F T T <---
190562024-02-27T21:51:23.829Z 1 020-039 170 170 170 310 310 310 F F F
190572024-02-27T21:51:23.829Z 2 040-059 173 173 173 313 313 313 F F F
190582024-02-27T21:51:23.829Z 3 060-079 167 167 167 304 304 304 F F F
190592024-02-27T21:51:23.829Z 4 080-099 170 170 170 309 309 309 F F F
190602024-02-27T21:51:23.829Z 5 100-119 172 176 176 312 320 320 F T T <---
190612024-02-27T21:51:23.829Z 6 120-139 167 176 176 304 318 318 F T T <---
190622024-02-27T21:51:23.829Z 7 140-159 168 168 168 305 305 305 F T T <---
190632024-02-27T21:51:23.829Z 8 160-179 174 176 176 314 320 320 F T T <---
190642024-02-27T21:51:23.829Z 9 180-199 170 170 170 310 310 310 F F F
190652024-02-27T21:51:23.830Z 10 200-219 172 176 176 312 316 316 T F F <---
190662024-02-27T21:51:23.830Z 11 220-239 172 172 172 312 312 312 F F F
190672024-02-27T21:51:23.830Z 12 240-259 166 166 166 302 302 302 F F F
190682024-02-27T21:51:23.830Z 13 260-279 170 170 170 310 310 310 F F F
190692024-02-27T21:51:23.830Z 14 280-299 175 175 175 315 315 315 F F F
190702024-02-27T21:51:23.830Z 15 300-319 170 170 170 310 310 310 F F F
190712024-02-27T21:51:23.830Z 16 320-339 175 176 176 315 318 318 F F F <---
190722024-02-27T21:51:23.830Z 17 340-359 175 175 175 315 315 315 F F F
190732024-02-27T21:51:23.830Z 18 360-379 175 175 175 315 315 315 F F F
190742024-02-27T21:51:23.830Z 19 380-399 175 175 175 315 315 315 F T T <---
190752024-02-27T21:51:23.830Z 20 400-419 174 174 174 314 314 314 F F F
190762024-02-27T21:51:23.830Z 21 420-439 171 171 171 311 311 311 F F F
190772024-02-27T21:51:23.830Z 22 440-459 172 172 172 312 312 312 F F F
190782024-02-27T21:51:23.830Z 23 460-479 175 175 175 315 315 315 F T T <---
190792024-02-27T21:51:23.830Z 24 480-499 174 174 174 314 314 314 F T T <---
190802024-02-27T21:51:23.831Z 25 500-519 172 176 176 312 318 318 F F F <---
190812024-02-27T21:51:23.831Z 26 520-539 175 176 176 315 317 317 F F F <---
190822024-02-27T21:51:23.831Z 27 540-559 166 176 176 303 317 317 F F F <---
190832024-02-27T21:51:23.831Z 28 560-579 175 175 175 315 315 315 F T T <---
190842024-02-27T21:51:23.831Z 29 580-599 174 174 174 314 314 314 F F F
190852024-02-27T21:51:23.831ZMax gen: 176, Max flush: 320
190862024-02-27T21:51:23.841ZError: Difference in extent metadata found!
190872024-02-27T21:51:23.841ZA Difference in extent metadata is expected here
190882024-02-27T21:51:23.841ZOn loop 88
190892024-02-27T21:51:23.841Z
190902024-02-27T21:51:23.841ZVerifying data now
190912024-02-27T21:51:28.532ZNo change after verify
190922024-02-27T21:51:28.532ZLoop: 88 Downstairs dump after verify (and repair):
190932024-02-27T21:51:28.546ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190942024-02-27T21:51:28.546Z 0 000-019 177 177 177 321 321 321 F F F
190952024-02-27T21:51:28.547Z 1 020-039 170 170 170 310 310 310 F F F
190962024-02-27T21:51:28.547Z 2 040-059 173 173 173 313 313 313 F F F
190972024-02-27T21:51:28.547Z 3 060-079 167 167 167 304 304 304 F F F
190982024-02-27T21:51:28.547Z 4 080-099 170 170 170 309 309 309 F F F
190992024-02-27T21:51:28.547Z 5 100-119 177 177 177 321 321 321 F F F
191002024-02-27T21:51:28.547Z 6 120-139 177 177 177 321 321 321 F F F
191012024-02-27T21:51:28.547Z 7 140-159 177 177 177 321 321 321 F F F
191022024-02-27T21:51:28.551Z 8 160-179 177 177 177 321 321 321 F F F
191032024-02-27T21:51:28.551Z 9 180-199 170 170 170 310 310 310 F F F
191042024-02-27T21:51:28.551Z 10 200-219 176 176 176 316 316 316 F F F
191052024-02-27T21:51:28.551Z 11 220-239 172 172 172 312 312 312 F F F
191062024-02-27T21:51:28.551Z 12 240-259 166 166 166 302 302 302 F F F
191072024-02-27T21:51:28.552Z 13 260-279 170 170 170 310 310 310 F F F
191082024-02-27T21:51:28.552Z 14 280-299 175 175 175 315 315 315 F F F
191092024-02-27T21:51:28.552Z 15 300-319 170 170 170 310 310 310 F F F
191102024-02-27T21:51:28.553Z 16 320-339 176 176 176 318 318 318 F F F
191112024-02-27T21:51:28.553Z 17 340-359 175 175 175 315 315 315 F F F
191122024-02-27T21:51:28.553Z 18 360-379 175 175 175 315 315 315 F F F
191132024-02-27T21:51:28.553Z 19 380-399 177 177 177 321 321 321 F F F
191142024-02-27T21:51:28.553Z 20 400-419 174 174 174 314 314 314 F F F
191152024-02-27T21:51:28.554Z 21 420-439 171 171 171 311 311 311 F F F
191162024-02-27T21:51:28.554Z 22 440-459 172 172 172 312 312 312 F F F
191172024-02-27T21:51:28.554Z 23 460-479 177 177 177 321 321 321 F F F
191182024-02-27T21:51:28.554Z 24 480-499 177 177 177 321 321 321 F F F
191192024-02-27T21:51:28.554Z 25 500-519 176 176 176 318 318 318 F F F
191202024-02-27T21:51:28.554Z 26 520-539 176 176 176 317 317 317 F F F
191212024-02-27T21:51:28.554Z 27 540-559 176 176 176 317 317 317 F F F
191222024-02-27T21:51:28.554Z 28 560-579 177 177 177 321 321 321 F F F
191232024-02-27T21:51:28.555Z 29 580-599 174 174 174 314 314 314 F F F
191242024-02-27T21:51:28.555ZMax gen: 177, Max flush: 321
191252024-02-27T21:51:28.555Z
191262024-02-27T21:51:28.555ZBegin loop 89
191272024-02-27T21:51:28.555ZDownstairs to restart: 0
191282024-02-27T21:51:28.575Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924140662Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862}
191292024-02-27T21:51:28.575Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.92441322Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862}
191302024-02-27T21:51:28.588Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924451225Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862}
191312024-02-27T21:51:28.588Z{"msg":"Crucible 37dc58bd-7125-427a-97ad-f968563fa1d8 has session id: a8f727f7-d456-49a4-ba05-bc7e766c063d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924481802Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191322024-02-27T21:51:28.588Z{"msg":"Upstairs opts: Upstairs UUID: 37dc58bd-7125-427a-97ad-f968563fa1d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924522987Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191332024-02-27T21:51:28.588Z{"msg":"Crucible stats registered with UUID: 37dc58bd-7125-427a-97ad-f968563fa1d8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924555153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191342024-02-27T21:51:28.589ZCrucible runtime is spawned
191352024-02-27T21:51:28.589Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924709385Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862}
191362024-02-27T21:51:28.589Z{"msg":"37dc58bd-7125-427a-97ad-f968563fa1d8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924851368Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191372024-02-27T21:51:28.589Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924898283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191382024-02-27T21:51:28.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924936598Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191392024-02-27T21:51:28.589Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924957996Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191402024-02-27T21:51:28.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.924994051Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191412024-02-27T21:51:28.589Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.92501015Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191422024-02-27T21:51:28.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.925033197Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191432024-02-27T21:51:28.589Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.925058524Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191442024-02-27T21:51:28.589Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.92508974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191452024-02-27T21:51:28.589Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.925123516Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191462024-02-27T21:51:28.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.925229294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191472024-02-27T21:51:28.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.925276318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191482024-02-27T21:51:28.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.925306215Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191492024-02-27T21:51:28.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.926049947Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191502024-02-27T21:51:28.590Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.926097202Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191512024-02-27T21:51:28.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.926140656Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191522024-02-27T21:51:28.590Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.926165514Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191532024-02-27T21:51:28.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.92619178Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191542024-02-27T21:51:28.590Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.926241014Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191552024-02-27T21:51:28.590Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.92670057Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191562024-02-27T21:51:28.590Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.926749825Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191572024-02-27T21:51:28.590Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.926784161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191582024-02-27T21:51:28.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927251146Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191592024-02-27T21:51:28.591Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927303979Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191602024-02-27T21:51:28.591Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927333746Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191612024-02-27T21:51:28.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927365312Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191622024-02-27T21:51:28.591Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927393919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191632024-02-27T21:51:28.591Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927411956Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191642024-02-27T21:51:28.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927435354Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191652024-02-27T21:51:28.591Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927452662Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191662024-02-27T21:51:28.591Z{"msg":"[0]R flush_numbers[0..12]: [321, 310, 313, 304, 309, 321, 321, 321, 321, 310, 316, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927476669Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191672024-02-27T21:51:28.591Z{"msg":"[0]R generation[0..12]: [177, 170, 173, 167, 170, 177, 177, 177, 177, 170, 176, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927495347Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191682024-02-27T21:51:28.591Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
191692024-02-27T21:51:28.592Z2024-02-27T21:51:26.927518534Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191702024-02-27T21:51:28.592Z{"msg":"[1]R flush_numbers[0..12]: [321, 310, 313, 304, 309, 321, 321, 321, 321, 310, 316, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927536442Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191712024-02-27T21:51:28.592Z{"msg":"[1]R generation[0..12]: [177, 170, 173, 167, 170, 177, 177, 177, 177, 170, 176, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927563129Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191722024-02-27T21:51:28.592Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927578457Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191732024-02-27T21:51:28.592Z{"msg":"[2]R flush_numbers[0..12]: [321, 310, 313, 304, 309, 321, 321, 321, 321, 310, 316, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927592455Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191742024-02-27T21:51:28.592Z{"msg":"[2]R generation[0..12]: [177, 170, 173, 167, 170, 177, 177, 177, 177, 170, 176, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927612273Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191752024-02-27T21:51:28.592Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927644199Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191762024-02-27T21:51:28.592Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927659937Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191772024-02-27T21:51:28.592Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927674166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191782024-02-27T21:51:28.592Z{"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927687304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191792024-02-27T21:51:28.593Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927718111Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"":"downstairs","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191802024-02-27T21:51:28.593Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927736058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191812024-02-27T21:51:28.593Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927756836Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"0","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191822024-02-27T21:51:28.593Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927770894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"1","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191832024-02-27T21:51:28.593Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927800991Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"client":"2","":"downstairs","session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191842024-02-27T21:51:28.593Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927820439Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191852024-02-27T21:51:28.593Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927833257Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191862024-02-27T21:51:28.593Z{"msg":"37dc58bd-7125-427a-97ad-f968563fa1d8 is now active with session: a8f727f7-d456-49a4-ba05-bc7e766c063d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927853664Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191872024-02-27T21:51:28.594Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927875002Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862,"session_id":"a8f727f7-d456-49a4-ba05-bc7e766c063d"}
191882024-02-27T21:51:28.594ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191892024-02-27T21:51:28.594Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:26.927908578Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862}
191902024-02-27T21:51:28.594ZLoading write count information from file "/tmp/test_repair_verify.data"
191912024-02-27T21:51:28.594ZRun Repair workload
191922024-02-27T21:51:28.594Z01/30 Write block 523 len 8192 data: 13 13
191932024-02-27T21:51:28.594Z02/30 Read block 54 len 24576
191942024-02-27T21:51:28.594Z03/30 Read block 74 len 24576
191952024-02-27T21:51:28.594Z04/30 Write block 124 len 24576 data: 18 13 11 11 14 13
191962024-02-27T21:51:28.595Z05/30 Read block 519 len 36864
191972024-02-27T21:51:28.595Z06/30 Read block 511 len 28672
191982024-02-27T21:51:28.595Z07/30 Read block 378 len 16384
191992024-02-27T21:51:28.595Z08/30 Write block 395 len 16384 data: 13 12 15 13
192002024-02-27T21:51:28.595Z09/30 Read block 493 len 36864
192012024-02-27T21:51:28.595Z10/30 Write block 196 len 32768 data: 10 12 10 12 13 11 10 10
192022024-02-27T21:51:28.595Z11/30 Read block 280 len 4096
192032024-02-27T21:51:28.595Z12/30 Write block 309 len 20480 data: 15 17 16 15 13
192042024-02-27T21:51:28.596Z13/30 Read block 179 len 16384
192052024-02-27T21:51:28.601Z14/30 Write block 586 len 8192 data: 15 13
192062024-02-27T21:51:28.601Z15/30 Read block 583 len 40960
192072024-02-27T21:51:28.601Z16/30 Read block 450 len 12288
192082024-02-27T21:51:28.601Z17/30 Read block 381 len 28672
192092024-02-27T21:51:28.601Z18/30 Write block 523 len 32768 data: 14 14 12 14 15 13 12 9
192102024-02-27T21:51:28.601Z19/30 Read block 55 len 32768
192112024-02-27T21:51:28.602Z20/30 Write block 517 len 4096 data: 19
192122024-02-27T21:51:28.602Z21/30 Read block 273 len 12288
192132024-02-27T21:51:28.602Z22/30 Write block 221 len 16384 data: 13 15 13 13
192142024-02-27T21:51:28.602Z23/30 Read block 193 len 36864
192152024-02-27T21:51:28.602Z24/30 Read block 541 len 40960
192162024-02-27T21:51:28.602Z25/30 Write block 428 len 24576 data: 11 12 13 13 13 13
192172024-02-27T21:51:28.602Z26/30 Flush
192182024-02-27T21:51:28.602Z27/30 Write block 329 len 40960 data: 14 14 14 14 16 17 21 17 17 18
192192024-02-27T21:51:28.602Z28/30 Write block 41 len 8192 data: 12 12
192202024-02-27T21:51:28.602Z29/30 Read block 592 len 12288
192212024-02-27T21:51:28.607Z30/30 Read block 518 len 28672
192222024-02-27T21:51:28.607Z----------------------------------------------------------------
192232024-02-27T21:51:28.607Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
192242024-02-27T21:51:28.607ZStates: Active Active Active
192252024-02-27T21:51:28.607ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192262024-02-27T21:51:28.607Z 1 Acked 1000 Write 2 Done Done Done false
192272024-02-27T21:51:28.607Z 2 Acked 1001 Read 6 Done Done Done false
192282024-02-27T21:51:28.607Z 3 Acked 1002 Read 6 Done Done Done false
192292024-02-27T21:51:28.607Z 4 Acked 1003 Write 6 Done Done Done false
192302024-02-27T21:51:28.607Z 5 Acked 1004 Read 9 Done Done Done false
192312024-02-27T21:51:28.607Z 6 Acked 1005 Read 7 Sent Done Done false
192322024-02-27T21:51:28.607Z 7 Acked 1006 Read 4 Sent Done Done false
192332024-02-27T21:51:28.607Z 8 Acked 1007 Write 4 Sent Done Done false
192342024-02-27T21:51:28.607Z 9 Acked 1008 Read 9 Sent Done Done false
192352024-02-27T21:51:28.608Z 10 Acked 1009 Write 8 Sent Done Done false
192362024-02-27T21:51:28.608Z 11 Acked 1010 Read 1 Sent Done Done false
192372024-02-27T21:51:28.608Z 12 Acked 1011 Write 5 Sent Done Done false
192382024-02-27T21:51:28.608Z 13 Acked 1012 Read 4 Sent Done Done false
192392024-02-27T21:51:28.608Z 14 Acked 1013 Write 2 Sent Done Done false
192402024-02-27T21:51:28.608Z 15 Acked 1014 Read 10 Sent Done Done false
192412024-02-27T21:51:28.608Z 16 Acked 1015 Read 3 Sent Done Done false
192422024-02-27T21:51:28.608Z 17 Acked 1016 Read 7 Sent Done Done false
192432024-02-27T21:51:28.608Z 18 Acked 1017 Write 8 Sent Done Done false
192442024-02-27T21:51:28.608Z 19 Acked 1018 Read 8 Sent Done Done false
192452024-02-27T21:51:28.608Z 20 Acked 1019 Write 1 Sent Done Done false
192462024-02-27T21:51:28.608Z 21 Acked 1020 Read 3 Sent Done Done false
192472024-02-27T21:51:28.608Z 22 Acked 1021 Write 4 Sent Done Done false
192482024-02-27T21:51:28.608Z 23 Acked 1022 Read 9 Sent Done Done false
192492024-02-27T21:51:28.608Z 24 Acked 1023 Read 10 Sent Done Done false
192502024-02-27T21:51:28.608Z 25 Acked 1024 Write 6 Sent Done Done false
192512024-02-27T21:51:28.608Z 26 Acked 1025 Flush 0 Sent Done Done false
192522024-02-27T21:51:28.608Z 27 Acked 1026 Write 10 Sent Done Done false
192532024-02-27T21:51:28.609Z 28 Acked 1027 Write 2 Sent Done Done false
192542024-02-27T21:51:28.609Z 29 Acked 1028 Read 3 Sent Done Done false
192552024-02-27T21:51:28.609Z 30 Acked 1029 Read 7 Sent Sent Done false
192562024-02-27T21:51:28.609Z STATES DS:0 DS:1 DS:2 TOTAL
192572024-02-27T21:51:28.609Z New 0 0 0 0
192582024-02-27T21:51:28.609Z Sent 25 1 0 26
192592024-02-27T21:51:28.609Z Done 5 29 30 64
192602024-02-27T21:51:28.609Z Skipped 0 0 0 0
192612024-02-27T21:51:28.609Z Error 0 0 0 0
192622024-02-27T21:51:28.609ZLast Flush: 0 1025 1025
192632024-02-27T21:51:28.609ZDownstairs last five completed:
192642024-02-27T21:51:28.609ZUpstairs last five completed: 30 29 28 27 26
192652024-02-27T21:51:28.609Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:26.957996134Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1862}
192662024-02-27T21:51:28.609ZWrote out file "/tmp/test_repair_verify.data"
192672024-02-27T21:51:28.610Z
192682024-02-27T21:51:28.614ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192692024-02-27T21:51:28.637ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192702024-02-27T21:51:28.637Z 0 000-019 177 177 177 321 321 321 F F F
192712024-02-27T21:51:28.638Z 1 020-039 170 170 170 310 310 310 F F F
192722024-02-27T21:51:28.638Z 2 040-059 173 173 173 313 313 313 F T T <---
192732024-02-27T21:51:28.638Z 3 060-079 167 167 167 304 304 304 F F F
192742024-02-27T21:51:28.638Z 4 080-099 170 170 170 309 309 309 F F F
192752024-02-27T21:51:28.654Z 5 100-119 177 177 177 321 321 321 F F F
192762024-02-27T21:51:28.654Z 6 120-139 177 178 178 321 322 322 T F F <---
192772024-02-27T21:51:28.655Z 7 140-159 177 177 177 321 321 321 F F F
192782024-02-27T21:51:28.655Z 8 160-179 177 177 177 321 321 321 F F F
192792024-02-27T21:51:28.655Z 9 180-199 170 178 178 310 322 322 F F F <---
192802024-02-27T21:51:28.655Z 10 200-219 176 178 178 316 322 322 F F F <---
192812024-02-27T21:51:28.655Z 11 220-239 172 178 178 312 322 322 F F F <---
192822024-02-27T21:51:28.655Z 12 240-259 166 166 166 302 302 302 F F F
192832024-02-27T21:51:28.655Z 13 260-279 170 170 170 310 310 310 F F F
192842024-02-27T21:51:28.655Z 14 280-299 175 175 175 315 315 315 F F F
192852024-02-27T21:51:28.655Z 15 300-319 170 178 178 310 322 322 F F F <---
192862024-02-27T21:51:28.655Z 16 320-339 176 176 176 318 318 318 F T T <---
192872024-02-27T21:51:28.655Z 17 340-359 175 175 175 315 315 315 F F F
192882024-02-27T21:51:28.655Z 18 360-379 175 175 175 315 315 315 F F F
192892024-02-27T21:51:28.655Z 19 380-399 177 178 178 321 322 322 F F F <---
192902024-02-27T21:51:28.655Z 20 400-419 174 174 174 314 314 314 F F F
192912024-02-27T21:51:28.655Z 21 420-439 171 178 178 311 322 322 F F F <---
192922024-02-27T21:51:28.655Z 22 440-459 172 172 172 312 312 312 F F F
192932024-02-27T21:51:28.655Z 23 460-479 177 177 177 321 321 321 F F F
192942024-02-27T21:51:28.655Z 24 480-499 177 177 177 321 321 321 F F F
192952024-02-27T21:51:28.655Z 25 500-519 176 178 178 318 322 322 F F F <---
192962024-02-27T21:51:28.656Z 26 520-539 176 178 178 317 322 322 T F F <---
192972024-02-27T21:51:28.656Z 27 540-559 176 176 176 317 317 317 F F F
192982024-02-27T21:51:28.656Z 28 560-579 177 177 177 321 321 321 F F F
192992024-02-27T21:51:28.656Z 29 580-599 174 178 178 314 322 322 F F F <---
193002024-02-27T21:51:28.656ZMax gen: 178, Max flush: 322
193012024-02-27T21:51:28.656ZError: Difference in extent metadata found!
193022024-02-27T21:51:28.656ZA Difference in extent metadata is expected here
193032024-02-27T21:51:28.656ZOn loop 89
193042024-02-27T21:51:28.656Z
193052024-02-27T21:51:28.656ZVerifying data now
193062024-02-27T21:51:34.244ZNo change after verify
193072024-02-27T21:51:34.244ZLoop: 89 Downstairs dump after verify (and repair):
193082024-02-27T21:51:34.258ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193092024-02-27T21:51:34.258Z 0 000-019 177 177 177 321 321 321 F F F
193102024-02-27T21:51:34.258Z 1 020-039 170 170 170 310 310 310 F F F
193112024-02-27T21:51:34.258Z 2 040-059 179 179 179 323 323 323 F F F
193122024-02-27T21:51:34.258Z 3 060-079 167 167 167 304 304 304 F F F
193132024-02-27T21:51:34.258Z 4 080-099 170 170 170 309 309 309 F F F
193142024-02-27T21:51:34.259Z 5 100-119 177 177 177 321 321 321 F F F
193152024-02-27T21:51:34.259Z 6 120-139 178 178 178 322 322 322 F F F
193162024-02-27T21:51:34.259Z 7 140-159 177 177 177 321 321 321 F F F
193172024-02-27T21:51:34.259Z 8 160-179 177 177 177 321 321 321 F F F
193182024-02-27T21:51:34.259Z 9 180-199 178 178 178 322 322 322 F F F
193192024-02-27T21:51:34.262Z 10 200-219 178 178 178 322 322 322 F F F
193202024-02-27T21:51:34.262Z 11 220-239 178 178 178 322 322 322 F F F
193212024-02-27T21:51:34.262Z 12 240-259 166 166 166 302 302 302 F F F
193222024-02-27T21:51:34.262Z 13 260-279 170 170 170 310 310 310 F F F
193232024-02-27T21:51:34.262Z 14 280-299 175 175 175 315 315 315 F F F
193242024-02-27T21:51:34.262Z 15 300-319 178 178 178 322 322 322 F F F
193252024-02-27T21:51:34.263Z 16 320-339 179 179 179 323 323 323 F F F
193262024-02-27T21:51:34.263Z 17 340-359 175 175 175 315 315 315 F F F
193272024-02-27T21:51:34.263Z 18 360-379 175 175 175 315 315 315 F F F
193282024-02-27T21:51:34.263Z 19 380-399 178 178 178 322 322 322 F F F
193292024-02-27T21:51:34.263Z 20 400-419 174 174 174 314 314 314 F F F
193302024-02-27T21:51:34.263Z 21 420-439 178 178 178 322 322 322 F F F
193312024-02-27T21:51:34.263Z 22 440-459 172 172 172 312 312 312 F F F
193322024-02-27T21:51:34.263Z 23 460-479 177 177 177 321 321 321 F F F
193332024-02-27T21:51:34.263Z 24 480-499 177 177 177 321 321 321 F F F
193342024-02-27T21:51:34.263Z 25 500-519 178 178 178 322 322 322 F F F
193352024-02-27T21:51:34.263Z 26 520-539 178 178 178 322 322 322 F F F
193362024-02-27T21:51:34.263Z 27 540-559 176 176 176 317 317 317 F F F
193372024-02-27T21:51:34.263Z 28 560-579 177 177 177 321 321 321 F F F
193382024-02-27T21:51:34.263Z 29 580-599 178 178 178 322 322 322 F F F
193392024-02-27T21:51:34.263ZMax gen: 179, Max flush: 323
193402024-02-27T21:51:34.263Z
193412024-02-27T21:51:34.264ZBegin loop 90
193422024-02-27T21:51:34.264ZDownstairs to restart: 1
193432024-02-27T21:51:34.281Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.637357149Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870}
193442024-02-27T21:51:34.281Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\nCrucible runtime is spawned
193452024-02-27T21:51:34.289Z git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.63768719Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870}
193462024-02-27T21:51:34.289Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.637734464Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870}
193472024-02-27T21:51:34.289Z{"msg":"Crucible 20707cc7-daa9-4bbe-97ce-f67192da4b48 has session id: 41eed752-5fbc-43f0-8a69-a495499adc8e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.637762381Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193482024-02-27T21:51:34.289Z{"msg":"Upstairs opts: Upstairs UUID: 20707cc7-daa9-4bbe-97ce-f67192da4b48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.637802476Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193492024-02-27T21:51:34.289Z{"msg":"Crucible stats registered with UUID: 20707cc7-daa9-4bbe-97ce-f67192da4b48","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.637819485Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193502024-02-27T21:51:34.289Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.637833893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870}
193512024-02-27T21:51:34.289Z{"msg":"20707cc7-daa9-4bbe-97ce-f67192da4b48 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.637918793Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193522024-02-27T21:51:34.289Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.63794525Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193532024-02-27T21:51:34.289Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.637964447Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193542024-02-27T21:51:34.289Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.637987005Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193552024-02-27T21:51:34.289Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.638008082Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193562024-02-27T21:51:34.289Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.638036689Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193572024-02-27T21:51:34.289Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.638064136Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193582024-02-27T21:51:34.289Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.638083343Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193592024-02-27T21:51:34.289Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.638161584Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193602024-02-27T21:51:34.289Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.63819925Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193612024-02-27T21:51:34.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.638340083Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193622024-02-27T21:51:34.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.638394267Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193632024-02-27T21:51:34.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.638423073Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193642024-02-27T21:51:34.289Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.639096374Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193652024-02-27T21:51:34.289Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.639180824Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193662024-02-27T21:51:34.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.63921772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193672024-02-27T21:51:34.290Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.639248576Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193682024-02-27T21:51:34.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.639627941Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193692024-02-27T21:51:34.290Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.639681255Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193702024-02-27T21:51:34.290Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.639751437Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193712024-02-27T21:51:34.290Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.639784603Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193722024-02-27T21:51:34.290Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640279345Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193732024-02-27T21:51:34.290Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.64031665Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193742024-02-27T21:51:34.290Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640333188Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193752024-02-27T21:51:34.290Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640347187Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193762024-02-27T21:51:34.290Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640365925Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193772024-02-27T21:51:34.290Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640413669Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193782024-02-27T21:51:34.290Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640456614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193792024-02-27T21:51:34.290Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.64083291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193802024-02-27T21:51:34.290Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640869555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193812024-02-27T21:51:34.295Z{"msg":"[0]R flush_numbers[0..12]: [321, 310, 323, 304, 309, 321, 322, 321, 321, 322, 322, 322]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640888153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193822024-02-27T21:51:34.295Z{"msg":"[0]R generation[0..12]: [177, 170, 179, 167, 170, 177, 178, 177, 177, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.64091427Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193832024-02-27T21:51:34.295Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640933378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193842024-02-27T21:51:34.295Z{"msg":"[1]R flush_numbers[0..12]: [321, 310, 323, 304, 309, 321, 322, 321, 321, 322, 322, 322]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640956775Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193852024-02-27T21:51:34.295Z{"msg":"[1]R generation[0..12]: [177, 170, 179, 167, 170, 177, 178, 177, 177, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.640973073Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193862024-02-27T21:51:34.295Z{"Wait for a query_work_queue command to finish before sending IO
193872024-02-27T21:51:34.296Zmsg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641001919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193882024-02-27T21:51:34.296Z{"msg":"[2]R flush_numbers[0..12]: [321, 310, 323, 304, 309, 321, 322, 321, 321, 322, 322, 322]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641018318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193892024-02-27T21:51:34.296Z{"msg":"[2]R generation[0..12]: [177, 170, 179, 167, 170, 177, 178, 177, 177, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641044345Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193902024-02-27T21:51:34.296Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641059943Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193912024-02-27T21:51:34.296Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641073851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193922024-02-27T21:51:34.296Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641087969Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193932024-02-27T21:51:34.296Z{"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641113796Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193942024-02-27T21:51:34.296Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641127975Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"":"downstairs","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193952024-02-27T21:51:34.296Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641149142Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193962024-02-27T21:51:34.296Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641162491Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"0","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193972024-02-27T21:51:34.296Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641176109Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"1","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193982024-02-27T21:51:34.296Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641213985Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"client":"2","":"downstairs","session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
193992024-02-27T21:51:34.296Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641228793Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
194002024-02-27T21:51:34.296Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641245671Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
194012024-02-27T21:51:34.296Z{"msg":"20707cc7-daa9-4bbe-97ce-f67192da4b48 is now active with session: 41eed752-5fbc-43f0-8a69-a495499adc8e","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641267978Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
194022024-02-27T21:51:34.296Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641280697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870,"session_id":"41eed752-5fbc-43f0-8a69-a495499adc8e"}
194032024-02-27T21:51:34.296Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:32.641293825Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870}
194042024-02-27T21:51:34.296ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
194052024-02-27T21:51:34.296ZLoading write count information from file "/tmp/test_repair_verify.data"
194062024-02-27T21:51:34.296ZRun Repair workload
194072024-02-27T21:51:34.296Z01/30 Write block 134 len 40960 data: 7 9 10 12 12 14 14 14 16 14
194082024-02-27T21:51:34.296Z02/30 Write block 171 len 4096 data: 14
194092024-02-27T21:51:34.296Z03/30 Flush
194102024-02-27T21:51:34.296Z04/30 Write block 311 len 36864 data: 17 16 14 15 11 17 16 16 18
194112024-02-27T21:51:34.296Z05/30 Read block 125 len 24576
194122024-02-27T21:51:34.297Z06/30 Write block 311 len 36864 data: 18 17 15 16 12 18 17 17 19
194132024-02-27T21:51:34.300Z07/30 Read block 128 len 16384
194142024-02-27T21:51:34.300Z08/30 Read block 588 len 28672
194152024-02-27T21:51:34.300Z09/30 Write block 159 len 24576 data: 15 15 13 12 13 11
194162024-02-27T21:51:34.300Z10/30 Read block 99 len 20480
194172024-02-27T21:51:34.300Z11/30 Flush
194182024-02-27T21:51:34.303Z12/30 Write block 172 len 16384 data: 14 12 16 14
194192024-02-27T21:51:34.303Z13/30 Read block 530 len 8192
194202024-02-27T21:51:34.303Z14/30 Write block 436 len 20480 data: 9 10 9 8 8
194212024-02-27T21:51:34.303Z15/30 Read block 531 len 4096
194222024-02-27T21:51:34.307Z16/30 Read block 124 len 8192
194232024-02-27T21:51:34.307Z17/30 Write block 107 len 20480 data: 19 20 17 16 20
194242024-02-27T21:51:34.307Z18/30 Write block 530 len 20480 data: 10 10 8 9 12
194252024-02-27T21:51:34.307Z19/30 Read block 225 len 8192
194262024-02-27T21:51:34.307Z20/30 Read block 42 len 20480
194272024-02-27T21:51:34.307Z21/30 Flush
194282024-02-27T21:51:34.310Z22/30 Write block 1 len 20480 data: 4 7 7 10 9
194292024-02-27T21:51:34.310Z23/30 Read block 342 len 28672
194302024-02-27T21:51:34.313Z24/30 Write block 233 len 12288 data: 19 20 21
194312024-02-27T21:51:34.313Z25/30 Read block 303 len 8192
194322024-02-27T21:51:34.313Z26/30 Read block 513 len 40960
194332024-02-27T21:51:34.317Z27/30 Read block 29 len 28672
194342024-02-27T21:51:34.317Z28/30 Read block 582 len 4096
194352024-02-27T21:51:34.317Z29/30 Read block 21 len 36864
194362024-02-27T21:51:34.317Z30/30 Write block 556 len 4096 data: 17
194372024-02-27T21:51:34.317Z----------------------------------------------------------------
194382024-02-27T21:51:34.317Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
194392024-02-27T21:51:34.317ZStates: Active Active Active
194402024-02-27T21:51:34.317ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
194412024-02-27T21:51:34.317Z 1 Acked 1000 Write 10 Done Done Done false
194422024-02-27T21:51:34.317Z 2 Acked 1001 Write 1 Done Done Done false
194432024-02-27T21:51:34.317Z 3 Acked 1002 Flush 0 Done Sent Done false
194442024-02-27T21:51:34.317Z 4 Acked 1003 Write 9 Done Sent Done false
194452024-02-27T21:51:34.317Z 5 Acked 1004 Read 6 Done Sent Done false
194462024-02-27T21:51:34.317Z 6 Acked 1005 Write 9 Done Sent Done false
194472024-02-27T21:51:34.317Z 7 Acked 1006 Read 4 Done Sent Done false
194482024-02-27T21:51:34.317Z 8 Acked 1007 Read 7 Done Sent Done false
194492024-02-27T21:51:34.317Z 9 Acked 1008 Write 6 Done Sent Done false
194502024-02-27T21:51:34.317Z 10 Acked 1009 Read 5 Done Sent Done false
194512024-02-27T21:51:34.317Z 11 Acked 1010 Flush 0 Done Sent Done false
194522024-02-27T21:51:34.317Z 12 Acked 1011 Write 4 Done Sent Done false
194532024-02-27T21:51:34.317Z 13 Acked 1012 Read 2 Done Sent Done false
194542024-02-27T21:51:34.318Z 14 Acked 1013 Write 5 Done Sent Done false
194552024-02-27T21:51:34.318Z 15 Acked 1014 Read 1 Done Sent Done false
194562024-02-27T21:51:34.318Z 16 Acked 1015 Read 2 Done Sent Done false
194572024-02-27T21:51:34.318Z 17 Acked 1016 Write 5 Done Sent Done false
194582024-02-27T21:51:34.318Z 18 Acked 1017 Write 5 Done Sent Done false
194592024-02-27T21:51:34.318Z 19 Acked 1018 Read 2 Done Sent Done false
194602024-02-27T21:51:34.318Z 20 Acked 1019 Read 5 Done Sent Done false
194612024-02-27T21:51:34.318Z 21 Acked 1020 Flush 0 Done Sent Done false
194622024-02-27T21:51:34.318Z 22 Acked 1021 Write 5 Done Sent Done false
194632024-02-27T21:51:34.318Z 23 Acked 1022 Read 7 Done Sent Done false
194642024-02-27T21:51:34.318Z 24 Acked 1023 Write 3 Done Sent Done false
194652024-02-27T21:51:34.318Z 25 Acked 1024 Read 2 Done Sent Done false
194662024-02-27T21:51:34.318Z 26 Acked 1025 Read 10 Done Sent Done false
194672024-02-27T21:51:34.318Z 27 Acked 1026 Read 7 Done Sent Done false
194682024-02-27T21:51:34.318Z 28 Acked 1027 Read 1 Done Sent Done false
194692024-02-27T21:51:34.318Z 29 Acked 1028 Read 9 Done Sent Done false
194702024-02-27T21:51:34.318Z 30 Acked 1029 Write 1 Sent Sent Sent false
194712024-02-27T21:51:34.318Z STATES DS:0 DS:1 DS:2 TOTAL
194722024-02-27T21:51:34.318Z New 0 0 0 0
194732024-02-27T21:51:34.318Z Sent 1 28 1 30
194742024-02-27T21:51:34.318Z Done 29 2 29 60
194752024-02-27T21:51:34.318Z Skipped 0 0 0 0
194762024-02-27T21:51:34.318Z Error 0 0 0 0
194772024-02-27T21:51:34.318ZLast Flush: 1020 0 1020
194782024-02-27T21:51:34.318ZDownstairs last five completed:
194792024-02-27T21:51:34.322ZUpstairs last five completed: 30 29 28 27 26
194802024-02-27T21:51:34.322Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:32.672838869Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1870}
194812024-02-27T21:51:34.322ZWrote out file "/tmp/test_repair_verify.data"
194822024-02-27T21:51:34.322Z
194832024-02-27T21:51:34.325ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194842024-02-27T21:51:34.340ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194852024-02-27T21:51:34.340Z 0 000-019 177 177 177 321 321 321 T F T <---
194862024-02-27T21:51:34.340Z 1 020-039 170 170 170 310 310 310 F F F
194872024-02-27T21:51:34.340Z 2 040-059 179 179 179 323 323 323 F F F
194882024-02-27T21:51:34.340Z 3 060-079 167 167 167 304 304 304 F F F
194892024-02-27T21:51:34.340Z 4 080-099 170 170 170 309 309 309 F F F
194902024-02-27T21:51:34.344Z 5 100-119 180 177 180 326 321 326 F F F <---
194912024-02-27T21:51:34.344Z 6 120-139 180 178 180 324 322 324 F T F <---
194922024-02-27T21:51:34.344Z 7 140-159 180 177 180 325 321 325 F T F <---
194932024-02-27T21:51:34.344Z 8 160-179 180 177 180 326 321 326 F T F <---
194942024-02-27T21:51:34.344Z 9 180-199 178 178 178 322 322 322 F F F
194952024-02-27T21:51:34.344Z 10 200-219 178 178 178 322 322 322 F F F
194962024-02-27T21:51:34.344Z 11 220-239 178 178 178 322 322 322 T F T <---
194972024-02-27T21:51:34.344Z 12 240-259 166 166 166 302 302 302 F F F
194982024-02-27T21:51:34.344Z 13 260-279 170 170 170 310 310 310 F F F
194992024-02-27T21:51:34.344Z 14 280-299 175 175 175 315 315 315 F F F
195002024-02-27T21:51:34.344Z 15 300-319 180 178 180 325 322 325 F F F <---
195012024-02-27T21:51:34.344Z 16 320-339 179 179 179 323 323 323 F F F
195022024-02-27T21:51:34.344Z 17 340-359 175 175 175 315 315 315 F F F
195032024-02-27T21:51:34.344Z 18 360-379 175 175 175 315 315 315 F F F
195042024-02-27T21:51:34.344Z 19 380-399 178 178 178 322 322 322 F F F
195052024-02-27T21:51:34.344Z 20 400-419 174 174 174 314 314 314 F F F
195062024-02-27T21:51:34.344Z 21 420-439 180 178 180 326 322 326 F F F <---
195072024-02-27T21:51:34.344Z 22 440-459 180 172 180 326 312 326 F F F <---
195082024-02-27T21:51:34.344Z 23 460-479 177 177 177 321 321 321 F F F
195092024-02-27T21:51:34.344Z 24 480-499 177 177 177 321 321 321 F F F
195102024-02-27T21:51:34.345Z 25 500-519 178 178 178 322 322 322 F F F
195112024-02-27T21:51:34.345Z 26 520-539 180 178 180 326 322 326 F F F <---
195122024-02-27T21:51:34.345Z 27 540-559 176 176 176 317 317 317 T F T <---
195132024-02-27T21:51:34.345Z 28 560-579 177 177 177 321 321 321 F F F
195142024-02-27T21:51:34.345Z 29 580-599 178 178 178 322 322 322 F F F
195152024-02-27T21:51:34.345ZMax gen: 180, Max flush: 326
195162024-02-27T21:51:34.345ZError: Difference in extent metadata found!
195172024-02-27T21:51:34.345ZA Difference in extent metadata is expected here
195182024-02-27T21:51:34.345ZOn loop 90
195192024-02-27T21:51:34.345Z
195202024-02-27T21:51:34.348ZVerifying data now
195212024-02-27T21:51:39.920ZNo change after verify
195222024-02-27T21:51:39.920ZLoop: 90 Downstairs dump after verify (and repair):
195232024-02-27T21:51:39.935ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195242024-02-27T21:51:39.935Z 0 000-019 181 181 181 327 327 327 F F F
195252024-02-27T21:51:39.935Z 1 020-039 170 170 170 310 310 310 F F F
195262024-02-27T21:51:39.935Z 2 040-059 179 179 179 323 323 323 F F F
195272024-02-27T21:51:39.935Z 3 060-079 167 167 167 304 304 304 F F F
195282024-02-27T21:51:39.936Z 4 080-099 170 170 170 309 309 309 F F F
195292024-02-27T21:51:39.936Z 5 100-119 180 180 180 326 326 326 F F F
195302024-02-27T21:51:39.936Z 6 120-139 180 180 180 324 324 324 F F F
195312024-02-27T21:51:39.936Z 7 140-159 180 180 180 325 325 325 F F F
195322024-02-27T21:51:39.939Z 8 160-179 180 180 180 326 326 326 F F F
195332024-02-27T21:51:39.939Z 9 180-199 178 178 178 322 322 322 F F F
195342024-02-27T21:51:39.940Z 10 200-219 178 178 178 322 322 322 F F F
195352024-02-27T21:51:39.940Z 11 220-239 181 181 181 327 327 327 F F F
195362024-02-27T21:51:39.940Z 12 240-259 166 166 166 302 302 302 F F F
195372024-02-27T21:51:39.940Z 13 260-279 170 170 170 310 310 310 F F F
195382024-02-27T21:51:39.940Z 14 280-299 175 175 175 315 315 315 F F F
195392024-02-27T21:51:39.940Z 15 300-319 180 180 180 325 325 325 F F F
195402024-02-27T21:51:39.940Z 16 320-339 179 179 179 323 323 323 F F F
195412024-02-27T21:51:39.940Z 17 340-359 175 175 175 315 315 315 F F F
195422024-02-27T21:51:39.940Z 18 360-379 175 175 175 315 315 315 F F F
195432024-02-27T21:51:39.940Z 19 380-399 178 178 178 322 322 322 F F F
195442024-02-27T21:51:39.940Z 20 400-419 174 174 174 314 314 314 F F F
195452024-02-27T21:51:39.940Z 21 420-439 180 180 180 326 326 326 F F F
195462024-02-27T21:51:39.940Z 22 440-459 180 180 180 326 326 326 F F F
195472024-02-27T21:51:39.941Z 23 460-479 177 177 177 321 321 321 F F F
195482024-02-27T21:51:39.941Z 24 480-499 177 177 177 321 321 321 F F F
195492024-02-27T21:51:39.941Z 25 500-519 178 178 178 322 322 322 F F F
195502024-02-27T21:51:39.941Z 26 520-539 180 180 180 326 326 326 F F F
195512024-02-27T21:51:39.941Z 27 540-559 181 181 181 327 327 327 F F F
195522024-02-27T21:51:39.941Z 28 560-579 177 177 177 321 321 321 F F F
195532024-02-27T21:51:39.941Z 29 580-599 178 178 178 322 322 322 F F F
195542024-02-27T21:51:39.941ZMax gen: 181, Max flush: 327
195552024-02-27T21:51:39.941Z
195562024-02-27T21:51:39.941ZBegin loop 91
195572024-02-27T21:51:39.941ZDownstairs to restart: 0
195582024-02-27T21:51:39.959Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.313688842Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878}
195592024-02-27T21:51:39.959Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.31396333Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878}
195602024-02-27T21:51:39.959Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.313981018Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878}
195612024-02-27T21:51:39.959Z{"msg":"Crucible ee544993-4ff6-494a-869f-5dd255e01bcf has session id: eb62ba89-c387-47bc-b64b-a29ae0ae0293","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314003635Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195622024-02-27T21:51:39.959Z{"msg":"Upstairs opts: Upstairs UUID: ee544993-4ff6-494a-869f-5dd255e01bcf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
195632024-02-27T21:51:39.959Z,"time":"2024-02-27T21:51:38.314043031Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195642024-02-27T21:51:39.964Z{"msg":"Crucible stats registered with UUID: ee544993-4ff6-494a-869f-5dd255e01bcf","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314081166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195652024-02-27T21:51:39.965Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314128401Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878}
195662024-02-27T21:51:39.965Z{"msg":"ee544993-4ff6-494a-869f-5dd255e01bcf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314264985Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195672024-02-27T21:51:39.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314298021Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195682024-02-27T21:51:39.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314339216Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195692024-02-27T21:51:39.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314366453Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195702024-02-27T21:51:39.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314401598Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195712024-02-27T21:51:39.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314444513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195722024-02-27T21:51:39.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314477809Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195732024-02-27T21:51:39.965Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314502706Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195742024-02-27T21:51:39.965Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314545032Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195752024-02-27T21:51:39.965Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314579217Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195762024-02-27T21:51:39.965Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.31464618Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195772024-02-27T21:51:39.965Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314734539Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195782024-02-27T21:51:39.965Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.314766685Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195792024-02-27T21:51:39.965Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.315413229Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195802024-02-27T21:51:39.965Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.315455114Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195812024-02-27T21:51:39.966Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.31549382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195822024-02-27T21:51:39.966Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.315533295Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195832024-02-27T21:51:39.966Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.315564181Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195842024-02-27T21:51:39.966Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.315632963Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195852024-02-27T21:51:39.966Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.315980323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195862024-02-27T21:51:39.966Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316075871Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195872024-02-27T21:51:39.966Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316123116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195882024-02-27T21:51:39.966Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316638385Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195892024-02-27T21:51:39.966Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316684029Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195902024-02-27T21:51:39.966Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316702777Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195912024-02-27T21:51:39.966Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316753921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195922024-02-27T21:51:39.966Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316772739Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195932024-02-27T21:51:39.966Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316786228Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195942024-02-27T21:51:39.966Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316807195Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195952024-02-27T21:51:39.966Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316821493Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195962024-02-27T21:51:39.966Z{"msg":"[0]R flush_numbers[0..12]: [327, 310, 323, 304, 309, 326, 324, 325, 326, 322, 322, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.31684937Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195972024-02-27T21:51:39.966Z{"msg":"[0]R generation[0..12]: [181, 170, 179, 167, 170, 180, 180, 180, 180, 178, 178, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316870898Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195982024-02-27T21:51:39.966Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316885176Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
195992024-02-27T21:51:39.967Z{"msg":"[1]R flush_numbers[0..12]: [327, 310, 323, 304, 309, 326, 324, 325, 326, 322, 322, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316906223Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196002024-02-27T21:51:39.967Z{"msg":"[1]R generation[0..12]: [181, 170, 179, 167, 170, 180, 180, 180, 180, 178, 178, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316920522Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196012024-02-27T21:51:39.967Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316941899Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196022024-02-27T21:51:39.967Z{"msg":"[2]R flush_numbers[0..12]: [327, 310, 323, 304, 309, 326, 324, 325, 326, 322, 322, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316958067Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196032024-02-27T21:51:39.967Z{"msg":"Wait for a query_work_queue command to finish before sending IO
196042024-02-27T21:51:39.967Z[2]R generation[0..12]: [181, 170, 179, 167, 170, 180, 180, 180, 180, 178, 178, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316982194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196052024-02-27T21:51:39.967Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.316997043Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196062024-02-27T21:51:39.967Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.31701755Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196072024-02-27T21:51:39.967Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317031299Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196082024-02-27T21:51:39.967Z{"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317051176Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196092024-02-27T21:51:39.967Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317064815Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"":"downstairs","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196102024-02-27T21:51:39.967Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317090572Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196112024-02-27T21:51:39.967Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.3171032Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"0","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196122024-02-27T21:51:39.967Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317123048Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"1","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196132024-02-27T21:51:39.967Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317137126Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"client":"2","":"downstairs","session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196142024-02-27T21:51:39.967Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317156974Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196152024-02-27T21:51:39.967Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317169582Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196162024-02-27T21:51:39.967Z{"msg":"ee544993-4ff6-494a-869f-5dd255e01bcf is now active with session: eb62ba89-c387-47bc-b64b-a29ae0ae0293","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.31718933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196172024-02-27T21:51:39.967Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317201539Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878,"session_id":"eb62ba89-c387-47bc-b64b-a29ae0ae0293"}
196182024-02-27T21:51:39.967Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:38.317225976Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878}
196192024-02-27T21:51:39.967ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
196202024-02-27T21:51:39.967ZLoading write count information from file "/tmp/test_repair_verify.data"
196212024-02-27T21:51:39.967ZRun Repair workload
196222024-02-27T21:51:39.968Z01/30 Write block 175 len 4096 data: 15
196232024-02-27T21:51:39.972Z02/30 Write block 18 len 40960 data: 17 16 15 15 11 9 14 12 11 11
196242024-02-27T21:51:39.972Z03/30 Write block 211 len 16384 data: 13 14 13 15
196252024-02-27T21:51:39.972Z04/30 Read block 152 len 32768
196262024-02-27T21:51:39.972Z05/30 Write block 324 len 28672 data: 14 13 11 14 14 15 15
196272024-02-27T21:51:39.972Z06/30 Read block 257 len 12288
196282024-02-27T21:51:39.972Z07/30 Write block 269 len 40960 data: 12 13 14 16 12 12 11 13 12 10
196292024-02-27T21:51:39.972Z08/30 Read block 30 len 36864
196302024-02-27T21:51:39.972Z09/30 Write block 118 len 8192 data: 14 15
196312024-02-27T21:51:39.987Z10/30 Read block 40 len 8192
196322024-02-27T21:51:39.987Z11/30 Write block 133 len 12288 data: 8 8 10
196332024-02-27T21:51:39.987Z12/30 Write block 430 len 24576 data: 14 14 14 14 15 12
196342024-02-27T21:51:39.988Z13/30 Read block 367 len 20480
196352024-02-27T21:51:39.988Z14/30 Write block 533 len 32768 data: 10 13 12 11 11 10 10 10
196362024-02-27T21:51:39.991Z15/30 Write block 469 len 8192 data: 14 11
196372024-02-27T21:51:39.991Z16/30 Write block 161 len 40960 data: 14 13 14 12 11 8 11 10 12 14
196382024-02-27T21:51:39.991Z17/30 Read block 1 len 8192
196392024-02-27T21:51:39.991Z18/30 Read block 505 len 40960
196402024-02-27T21:51:39.991Z19/30 Read block 301 len 8192
196412024-02-27T21:51:39.991Z20/30 Write block 197 len 12288 data: 13 11 13
196422024-02-27T21:51:39.992Z21/30 Write block 197 len 8192 data: 14 12
196432024-02-27T21:51:39.992Z22/30 Write block 110 len 28672 data: 17 21 17 17 16 14 12
196442024-02-27T21:51:39.992Z23/30 Read block 503 len 12288
196452024-02-27T21:51:39.992Z24/30 Read block 438 len 20480
196462024-02-27T21:51:39.992Z25/30 Flush
196472024-02-27T21:51:39.992Z26/30 Write block 492 len 4096 data: 13
196482024-02-27T21:51:39.992Z27/30 Write block 571 len 28672 data: 10 12 13 11 11 11 13
196492024-02-27T21:51:39.992Z28/30 Read block 95 len 28672
196502024-02-27T21:51:39.996Z29/30 Read block 291 len 28672
196512024-02-27T21:51:39.996Z30/30 Read block 509 len 32768
196522024-02-27T21:51:39.996Z----------------------------------------------------------------
196532024-02-27T21:51:39.996Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
196542024-02-27T21:51:39.997ZStates: Active Active Active
196552024-02-27T21:51:39.997ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196562024-02-27T21:51:39.997Z 1 Acked 1000 Write 1 Sent Done Done false
196572024-02-27T21:51:39.997Z 2 Acked 1001 Write 10 Sent Done Done false
196582024-02-27T21:51:39.997Z 3 Acked 1002 Write 4 Sent Done Done false
196592024-02-27T21:51:39.997Z 4 Acked 1003 Read 8 Sent Done Done false
196602024-02-27T21:51:39.997Z 5 Acked 1004 Write 7 Sent Done Done false
196612024-02-27T21:51:39.997Z 6 Acked 1005 Read 3 Sent Done Done false
196622024-02-27T21:51:39.997Z 7 Acked 1006 Write 10 Sent Done Done false
196632024-02-27T21:51:39.997Z 8 Acked 1007 Read 9 Sent Done Done false
196642024-02-27T21:51:39.997Z 9 Acked 1008 Write 2 Sent Done Done false
196652024-02-27T21:51:39.997Z 10 Acked 1009 Read 2 Sent Done Done false
196662024-02-27T21:51:39.998Z 11 Acked 1010 Write 3 Sent Done Done false
196672024-02-27T21:51:39.998Z 12 Acked 1011 Write 6 Sent Done Done false
196682024-02-27T21:51:39.998Z 13 Acked 1012 Read 5 Sent Done Done false
196692024-02-27T21:51:39.998Z 14 Acked 1013 Write 8 Sent Done Done false
196702024-02-27T21:51:39.998Z 15 Acked 1014 Write 2 Sent Done Done false
196712024-02-27T21:51:39.998Z 16 Acked 1015 Write 10 Sent Done Done false
196722024-02-27T21:51:39.998Z 17 Acked 1016 Read 2 Sent Done Done false
196732024-02-27T21:51:39.998Z 18 Acked 1017 Read 10 Sent Done Done false
196742024-02-27T21:51:39.998Z 19 Acked 1018 Read 2 Sent Done Done false
196752024-02-27T21:51:39.998Z 20 Acked 1019 Write 3 Sent Done Done false
196762024-02-27T21:51:39.998Z 21 Acked 1020 Write 2 Sent Done Done false
196772024-02-27T21:51:39.998Z 22 Acked 1021 Write 7 Sent Done Done false
196782024-02-27T21:51:39.998Z 23 Acked 1022 Read 3 Sent Done Done false
196792024-02-27T21:51:39.999Z 24 Acked 1023 Read 5 Sent Done Done false
196802024-02-27T21:51:39.999Z 25 Acked 1024 Flush 0 Sent Done Done false
196812024-02-27T21:51:39.999Z 26 Acked 1025 Write 1 Sent Done Done false
196822024-02-27T21:51:39.999Z 27 Acked 1026 Write 7 Sent Done Done false
196832024-02-27T21:51:39.999Z 28 Acked 1027 Read 7 Sent Done Done false
196842024-02-27T21:51:39.999Z 29 Acked 1028 Read 7 Sent Done Done false
196852024-02-27T21:51:39.999Z 30 Acked 1029 Read 8 Sent Sent Done false
196862024-02-27T21:51:39.999Z STATES DS:0 DS:1 DS:2 TOTAL
196872024-02-27T21:51:39.999Z New 0 0 0 0
196882024-02-27T21:51:39.999Z Sent 30 1 0 31
196892024-02-27T21:51:39.999Z Done 0 29 30 59
196902024-02-27T21:51:39.999Z Skipped 0 0 0 0
196912024-02-27T21:51:39.999Z Error 0 0 0 0
196922024-02-27T21:51:40.000ZLast Flush: 0 1024 1024
196932024-02-27T21:51:40.000ZDownstairs last five completed:
196942024-02-27T21:51:40.000ZUpstairs last five completed: 30 29 28 27 26
196952024-02-27T21:51:40.000Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:38.348738854Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1878}
196962024-02-27T21:51:40.000ZWrote out file "/tmp/test_repair_verify.data"
196972024-02-27T21:51:40.004Z
196982024-02-27T21:51:40.004ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196992024-02-27T21:51:40.018ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197002024-02-27T21:51:40.018Z 0 000-019 181 182 182 327 328 328 F F F <---
197012024-02-27T21:51:40.018Z 1 020-039 170 182 182 310 328 328 F F F <---
197022024-02-27T21:51:40.018Z 2 040-059 179 179 179 323 323 323 F F F
197032024-02-27T21:51:40.018Z 3 060-079 167 167 167 304 304 304 F F F
197042024-02-27T21:51:40.018Z 4 080-099 170 170 170 309 309 309 F F F
197052024-02-27T21:51:40.018Z 5 100-119 180 182 182 326 328 328 F F F <---
197062024-02-27T21:51:40.019Z 6 120-139 180 182 182 324 328 328 F F F <---
197072024-02-27T21:51:40.019Z 7 140-159 180 180 180 325 325 325 F F F
197082024-02-27T21:51:40.019Z 8 160-179 180 182 182 326 328 328 F F F <---
197092024-02-27T21:51:40.019Z 9 180-199 178 182 182 322 328 328 F F F <---
197102024-02-27T21:51:40.022Z 10 200-219 178 182 182 322 328 328 F F F <---
197112024-02-27T21:51:40.023Z 11 220-239 181 181 181 327 327 327 F F F
197122024-02-27T21:51:40.023Z 12 240-259 166 166 166 302 302 302 F F F
197132024-02-27T21:51:40.023Z 13 260-279 170 182 182 310 328 328 F F F <---
197142024-02-27T21:51:40.023Z 14 280-299 175 175 175 315 315 315 F F F
197152024-02-27T21:51:40.023Z 15 300-319 180 180 180 325 325 325 F F F
197162024-02-27T21:51:40.023Z 16 320-339 179 182 182 323 328 328 F F F <---
197172024-02-27T21:51:40.023Z 17 340-359 175 175 175 315 315 315 F F F
197182024-02-27T21:51:40.023Z 18 360-379 175 175 175 315 315 315 F F F
197192024-02-27T21:51:40.023Z 19 380-399 178 178 178 322 322 322 F F F
197202024-02-27T21:51:40.023Z 20 400-419 174 174 174 314 314 314 F F F
197212024-02-27T21:51:40.023Z 21 420-439 180 182 182 326 328 328 F F F <---
197222024-02-27T21:51:40.023Z 22 440-459 180 180 180 326 326 326 F F F
197232024-02-27T21:51:40.024Z 23 460-479 177 182 182 321 328 328 F F F <---
197242024-02-27T21:51:40.024Z 24 480-499 177 177 177 321 321 321 F T T <---
197252024-02-27T21:51:40.024Z 25 500-519 178 178 178 322 322 322 F F F
197262024-02-27T21:51:40.024Z 26 520-539 180 182 182 326 328 328 F F F <---
197272024-02-27T21:51:40.024Z 27 540-559 181 182 182 327 328 328 F F F <---
197282024-02-27T21:51:40.024Z 28 560-579 177 177 177 321 321 321 F T T <---
197292024-02-27T21:51:40.024Z 29 580-599 178 178 178 322 322 322 F F F
197302024-02-27T21:51:40.024ZMax gen: 182, Max flush: 328
197312024-02-27T21:51:40.024ZError: Difference in extent metadata found!
197322024-02-27T21:51:40.024ZA Difference in extent metadata is expected here
197332024-02-27T21:51:40.024ZOn loop 91
197342024-02-27T21:51:40.024Z
197352024-02-27T21:51:40.031ZVerifying data now
197362024-02-27T21:51:44.777ZNo change after verify
197372024-02-27T21:51:44.778ZLoop: 91 Downstairs dump after verify (and repair):
197382024-02-27T21:51:44.797ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197392024-02-27T21:51:44.797Z 0 000-019 182 182 182 328 328 328 F F F
197402024-02-27T21:51:44.798Z 1 020-039 182 182 182 328 328 328 F F F
197412024-02-27T21:51:44.798Z 2 040-059 179 179 179 323 323 323 F F F
197422024-02-27T21:51:44.798Z 3 060-079 167 167 167 304 304 304 F F F
197432024-02-27T21:51:44.798Z 4 080-099 170 170 170 309 309 309 F F F
197442024-02-27T21:51:44.799Z 5 100-119 182 182 182 328 328 328 F F F
197452024-02-27T21:51:44.799Z 6 120-139 182 182 182 328 328 328 F F F
197462024-02-27T21:51:44.799Z 7 140-159 180 180 180 325 325 325 F F F
197472024-02-27T21:51:44.807Z 8 160-179 182 182 182 328 328 328 F F F
197482024-02-27T21:51:44.807Z 9 180-199 182 182 182 328 328 328 F F F
197492024-02-27T21:51:44.807Z 10 200-219 182 182 182 328 328 328 F F F
197502024-02-27T21:51:44.807Z 11 220-239 181 181 181 327 327 327 F F F
197512024-02-27T21:51:44.808Z 12 240-259 166 166 166 302 302 302 F F F
197522024-02-27T21:51:44.808Z 13 260-279 182 182 182 328 328 328 F F F
197532024-02-27T21:51:44.808Z 14 280-299 175 175 175 315 315 315 F F F
197542024-02-27T21:51:44.808Z 15 300-319 180 180 180 325 325 325 F F F
197552024-02-27T21:51:44.809Z 16 320-339 182 182 182 328 328 328 F F F
197562024-02-27T21:51:44.809Z 17 340-359 175 175 175 315 315 315 F F F
197572024-02-27T21:51:44.809Z 18 360-379 175 175 175 315 315 315 F F F
197582024-02-27T21:51:44.809Z 19 380-399 178 178 178 322 322 322 F F F
197592024-02-27T21:51:44.809Z 20 400-419 174 174 174 314 314 314 F F F
197602024-02-27T21:51:44.810Z 21 420-439 182 182 182 328 328 328 F F F
197612024-02-27T21:51:44.810Z 22 440-459 180 180 180 326 326 326 F F F
197622024-02-27T21:51:44.810Z 23 460-479 182 182 182 328 328 328 F F F
197632024-02-27T21:51:44.810Z 24 480-499 183 183 183 329 329 329 F F F
197642024-02-27T21:51:44.811Z 25 500-519 178 178 178 322 322 322 F F F
197652024-02-27T21:51:44.811Z 26 520-539 182 182 182 328 328 328 F F F
197662024-02-27T21:51:44.811Z 27 540-559 182 182 182 328 328 328 F F F
197672024-02-27T21:51:44.812Z 28 560-579 183 183 183 329 329 329 F F F
197682024-02-27T21:51:44.812Z 29 580-599 178 178 178 322 322 322 F F F
197692024-02-27T21:51:44.812ZMax gen: 183, Max flush: 329
197702024-02-27T21:51:44.813Z
197712024-02-27T21:51:44.813ZBegin loop 92
197722024-02-27T21:51:44.813ZDownstairs to restart: 1
197732024-02-27T21:51:44.845Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.168251382Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886}
197742024-02-27T21:51:44.845Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.168581283Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886}
197752024-02-27T21:51:44.846Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.16860407Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886}
197762024-02-27T21:51:44.846Z{"msg":"Crucible 49d3dfae-bbd0-4bab-bca6-795bcf1ecee6 has session id: 69a50488-319a-45e9-96e5-4b9403f26318","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.168618499Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197772024-02-27T21:51:44.846Z{"msg":"Upstairs opts: Upstairs UUID: 49d3dfae-bbd0-4bab-bca6-795bcf1ecee6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.168645975Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197782024-02-27T21:51:44.846Z{Crucible runtime is spawned
197792024-02-27T21:51:44.847Z"msg":"Crucible stats registered with UUID: 49d3dfae-bbd0-4bab-bca6-795bcf1ecee6","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.16868546Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197802024-02-27T21:51:44.848Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.168784659Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886}
197812024-02-27T21:51:44.848Z{"msg":"49d3dfae-bbd0-4bab-bca6-795bcf1ecee6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.168989225Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197822024-02-27T21:51:44.848Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169022001Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197832024-02-27T21:51:44.849Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169091863Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197842024-02-27T21:51:44.849Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169132778Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197852024-02-27T21:51:44.851Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169160375Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197862024-02-27T21:51:44.851Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.16919676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197872024-02-27T21:51:44.851Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169225327Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197882024-02-27T21:51:44.851Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169264042Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197892024-02-27T21:51:44.851Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169291959Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197902024-02-27T21:51:44.852Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169316056Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197912024-02-27T21:51:44.852Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.16945011Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197922024-02-27T21:51:44.852Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169654576Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197932024-02-27T21:51:44.852Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.169714279Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197942024-02-27T21:51:44.853Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.170119692Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197952024-02-27T21:51:44.853Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.170155657Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197962024-02-27T21:51:44.854Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.170278543Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197972024-02-27T21:51:44.854Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.170328697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197982024-02-27T21:51:44.855Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.170344335Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
197992024-02-27T21:51:44.855Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.17039055Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198002024-02-27T21:51:44.855Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.170653629Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198012024-02-27T21:51:44.855Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.170870053Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198022024-02-27T21:51:44.855Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171053242Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198032024-02-27T21:51:44.855Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171106855Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198042024-02-27T21:51:44.855Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171131932Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198052024-02-27T21:51:44.855Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171163909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198062024-02-27T21:51:44.855Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171385022Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198072024-02-27T21:51:44.855Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171423788Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198082024-02-27T21:51:44.855Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171449485Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198092024-02-27T21:51:44.856Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171585719Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198102024-02-27T21:51:44.856Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171634513Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198112024-02-27T21:51:44.856Z{"msg":"[0]R flush_numbers[0..12]: [328, 328, 323, 304, 309, 328, 328, 325, 328, 328, 328, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.17166624Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198122024-02-27T21:51:44.856Z{"msg":"[0]R generation[0..12]: [182, 182, 179, 167, 170, 182, 182, 180, 182, 182, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171684227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198132024-02-27T21:51:44.856Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171707075Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198142024-02-27T21:51:44.857Z{"msg":"[1]R flush_numbers[0..12]: [328, 328, 323, 304, 309, 328, 328, 325, 328, 328, 328, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171721973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198152024-02-27T21:51:44.857Z{"msg":"[1]R generation[0..12]: [182, 182, 179, 167, 170, 182, 182, 180, 182, 182, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.17174322Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198162024-02-27T21:51:44.857Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171759338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
198172024-02-27T21:51:44.857Z69a50488-319a-45e9-96e5-4b9403f26318"}
198182024-02-27T21:51:44.858Z{"msg":"[2]R flush_numbers[0..12]: [328, 328, 323, 304, 309, 328, 328, 325, 328, 328, 328, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171784255Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198192024-02-27T21:51:44.858Z{"msg":"[2]R generation[0..12]: [182, 182, 179, 167, 170, 182, 182, 180, 182, 182, 182, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171798194Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198202024-02-27T21:51:44.858Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171812952Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198212024-02-27T21:51:44.858Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171826361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198222024-02-27T21:51:44.858Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171847918Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198232024-02-27T21:51:44.858Z{"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171861277Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198242024-02-27T21:51:44.858Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171875035Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"":"downstairs","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198252024-02-27T21:51:44.858Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171888733Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198262024-02-27T21:51:44.859Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171911411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"0","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198272024-02-27T21:51:44.859Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171928278Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"1","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198282024-02-27T21:51:44.859Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171942907Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"client":"2","":"downstairs","session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198292024-02-27T21:51:44.859Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171956275Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198302024-02-27T21:51:44.859Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171969494Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198312024-02-27T21:51:44.859Z{"msg":"49d3dfae-bbd0-4bab-bca6-795bcf1ecee6 is now active with session: 69a50488-319a-45e9-96e5-4b9403f26318","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.171982822Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198322024-02-27T21:51:44.859Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.17200218Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886,"session_id":"69a50488-319a-45e9-96e5-4b9403f26318"}
198332024-02-27T21:51:44.860Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:43.172016448Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886}
198342024-02-27T21:51:44.860ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
198352024-02-27T21:51:44.860ZLoading write count information from file "/tmp/test_repair_verify.data"
198362024-02-27T21:51:44.860ZRun Repair workload
198372024-02-27T21:51:44.860Z01/30 Write block 576 len 4096 data: 12
198382024-02-27T21:51:44.876Z02/30 Write block 551 len 36864 data: 21 17 21 21 19 18 21 17 16
198392024-02-27T21:51:44.876Z03/30 Flush
198402024-02-27T21:51:44.877Z04/30 Write block 205 len 40960 data: 6 6 9 11 13 13 14 15 14 16
198412024-02-27T21:51:44.877Z05/30 Read block 353 len 4096
198422024-02-27T21:51:44.877Z06/30 Read block 112 len 20480
198432024-02-27T21:51:44.877Z07/30 Write block 3 len 28672 data: 8 11 10 11 12 13 15
198442024-02-27T21:51:44.877Z08/30 Write block 39 len 4096 data: 11
198452024-02-27T21:51:44.877Z09/30 Write block 407 len 16384 data: 9 12 11 9
198462024-02-27T21:51:44.878Z10/30 Read block 584 len 12288
198472024-02-27T21:51:44.878Z11/30 Read block 333 len 36864
198482024-02-27T21:51:44.878Z12/30 Write block 572 len 20480 data: 13 14 12 12 13
198492024-02-27T21:51:44.878Z13/30 Write block 404 len 28672 data: 14 12 12 10 13 12 10
198502024-02-27T21:51:44.878Z14/30 Read block 506 len 12288
198512024-02-27T21:51:44.878Z15/30 Flush
198522024-02-27T21:51:44.879Z16/30 Write block 64 len 4096 data: 11
198532024-02-27T21:51:44.879Z17/30 Write block 409 len 4096 data: 13
198542024-02-27T21:51:44.879Z18/30 Read block 355 len 12288
198552024-02-27T21:51:44.879Z19/30 Write block 451 len 16384 data: 11 9 10 9
198562024-02-27T21:51:44.880Z20/30 Read block 141 len 28672
198572024-02-27T21:51:44.880Z21/30 Write block 592 len 32768 data: 14 15 12 11 12 5 6 4
198582024-02-27T21:51:44.880Z22/30 Write block 368 len 40960 data: 12 8 9 9 7 8 10 10 10 12
198592024-02-27T21:51:44.881Z23/30 Read block 308 len 8192
198602024-02-27T21:51:44.881Z24/30 Read block 13 len 28672
198612024-02-27T21:51:44.881Z25/30 Write block 218 len 28672 data: 18 16 14 14 16 14 14
198622024-02-27T21:51:44.882Z26/30 Read block 465 len 28672
198632024-02-27T21:51:44.882Z27/30 Read block 14 len 40960
198642024-02-27T21:51:44.882Z28/30 Write block 518 len 8192 data: 15 14
198652024-02-27T21:51:44.882Z29/30 Read block 2 len 8192
198662024-02-27T21:51:44.882Z30/30 Write block 276 len 28672 data: 14 13 11 10 9 8 8
198672024-02-27T21:51:44.882Z----------------------------------------------------------------
198682024-02-27T21:51:44.883Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
198692024-02-27T21:51:44.883ZStates: Active Active Active
198702024-02-27T21:51:44.884ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198712024-02-27T21:51:44.884Z 1 Acked 1000 Write 1 Done Done Done false
198722024-02-27T21:51:44.884Z 2 Acked 1001 Write 9 Done Sent Done false
198732024-02-27T21:51:44.884Z 3 Acked 1002 Flush 0 Done Sent Done false
198742024-02-27T21:51:44.885Z 4 Acked 1003 Write 10 Done Sent Done false
198752024-02-27T21:51:44.885Z 5 Acked 1004 Read 1 Done Sent Done false
198762024-02-27T21:51:44.885Z 6 Acked 1005 Read 5 Done Sent Done false
198772024-02-27T21:51:44.885Z 7 Acked 1006 Write 7 Done Sent Done false
198782024-02-27T21:51:44.885Z 8 Acked 1007 Write 1 Done Sent Done false
198792024-02-27T21:51:44.886Z 9 Acked 1008 Write 4 Done Sent Done false
198802024-02-27T21:51:44.886Z 10 Acked 1009 Read 3 Done Sent Done false
198812024-02-27T21:51:44.886Z 11 Acked 1010 Read 9 Done Sent Done false
198822024-02-27T21:51:44.886Z 12 Acked 1011 Write 5 Done Sent Done false
198832024-02-27T21:51:44.887Z 13 Acked 1012 Write 7 Done Sent Done false
198842024-02-27T21:51:44.887Z 14 Acked 1013 Read 3 Done Sent Done false
198852024-02-27T21:51:44.887Z 15 Acked 1014 Flush 0 Done Sent Done false
198862024-02-27T21:51:44.887Z 16 Acked 1015 Write 1 Done Sent Done false
198872024-02-27T21:51:44.888Z 17 Acked 1016 Write 1 Done Sent Done false
198882024-02-27T21:51:44.888Z 18 Acked 1017 Read 3 Done Sent Done false
198892024-02-27T21:51:44.888Z 19 Acked 1018 Write 4 Done Sent Done false
198902024-02-27T21:51:44.888Z 20 Acked 1019 Read 7 Done Sent Done false
198912024-02-27T21:51:44.889Z 21 Acked 1020 Write 8 Done Sent Done false
198922024-02-27T21:51:44.889Z 22 Acked 1021 Write 10 Done Sent Done false
198932024-02-27T21:51:44.889Z 23 Acked 1022 Read 2 Done Sent Done false
198942024-02-27T21:51:44.890Z 24 Acked 1023 Read 7 Done Sent Done false
198952024-02-27T21:51:44.890Z 25 Acked 1024 Write 7 Done Sent Done false
198962024-02-27T21:51:44.890Z 26 Acked 1025 Read 7 Done Sent Done false
198972024-02-27T21:51:44.890Z 27 Acked 1026 Read 10 Done Sent Done false
198982024-02-27T21:51:44.891Z 28 Acked 1027 Write 2 Done Sent Done false
198992024-02-27T21:51:44.891Z 29 Acked 1028 Read 2 Done Sent Done false
199002024-02-27T21:51:44.892Z 30 Acked 1029 Write 7 Sent Sent Sent false
199012024-02-27T21:51:44.892Z STATES DS:0 DS:1 DS:2 TOTAL
199022024-02-27T21:51:44.892Z New 0 0 0 0
199032024-02-27T21:51:44.892Z Sent 1 29 1 31
199042024-02-27T21:51:44.893Z Done 29 1 29 59
199052024-02-27T21:51:44.893Z Skipped 0 0 0 0
199062024-02-27T21:51:44.893Z Error 0 0 0 0
199072024-02-27T21:51:44.893ZLast Flush: 1014 0 1014
199082024-02-27T21:51:44.894ZDownstairs last five completed:
199092024-02-27T21:51:44.894ZUpstairs last five completed: 30 29 28 27 26
199102024-02-27T21:51:44.894Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:43.201892719Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1886}
199112024-02-27T21:51:44.894ZWrote out file "/tmp/test_repair_verify.data"
199122024-02-27T21:51:44.894Z
199132024-02-27T21:51:44.895ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
199142024-02-27T21:51:44.981ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199152024-02-27T21:51:44.981Z 0 000-019 184 182 184 331 328 331 F F F <---
199162024-02-27T21:51:44.981Z 1 020-039 184 182 184 331 328 331 F F F <---
199172024-02-27T21:51:44.981Z 2 040-059 179 179 179 323 323 323 F F F
199182024-02-27T21:51:44.982Z 3 060-079 167 167 167 304 304 304 T F T <---
199192024-02-27T21:51:44.982Z 4 080-099 170 170 170 309 309 309 F F F
199202024-02-27T21:51:44.982Z 5 100-119 182 182 182 328 328 328 F F F
199212024-02-27T21:51:44.982Z 6 120-139 182 182 182 328 328 328 F F F
199222024-02-27T21:51:44.982Z 7 140-159 180 180 180 325 325 325 F F F
199232024-02-27T21:51:44.982Z 8 160-179 182 182 182 328 328 328 F F F
199242024-02-27T21:51:44.983Z 9 180-199 182 182 182 328 328 328 F F F
199252024-02-27T21:51:44.983Z 10 200-219 184 182 184 331 328 331 T F T <---
199262024-02-27T21:51:44.983Z 11 220-239 181 181 181 327 327 327 T F T <---
199272024-02-27T21:51:44.983Z 12 240-259 166 166 166 302 302 302 F F F
199282024-02-27T21:51:44.983Z 13 260-279 182 182 182 328 328 328 T F T <---
199292024-02-27T21:51:44.983Z 14 280-299 175 175 175 315 315 315 T F T <---
199302024-02-27T21:51:44.983Z 15 300-319 180 180 180 325 325 325 F F F
199312024-02-27T21:51:44.984Z 16 320-339 182 182 182 328 328 328 F F F
199322024-02-27T21:51:44.984Z 17 340-359 175 175 175 315 315 315 F F F
199332024-02-27T21:51:44.984Z 18 360-379 175 175 175 315 315 315 T F T <---
199342024-02-27T21:51:44.984Z 19 380-399 178 178 178 322 322 322 F F F
199352024-02-27T21:51:44.985Z 20 400-419 184 174 184 331 314 331 T F T <---
199362024-02-27T21:51:44.985Z 21 420-439 182 182 182 328 328 328 F F F
199372024-02-27T21:51:44.985Z 22 440-459 180 180 180 326 326 326 T F T <---
199382024-02-27T21:51:44.985Z 23 460-479 182 182 182 328 328 328 F F F
199392024-02-27T21:51:44.986Z 24 480-499 183 183 183 329 329 329 F F F
199402024-02-27T21:51:44.986Z 25 500-519 178 178 178 322 322 322 T F T <---
199412024-02-27T21:51:44.986Z 26 520-539 182 182 182 328 328 328 F F F
199422024-02-27T21:51:44.986Z 27 540-559 184 182 184 330 328 330 F F F <---
199432024-02-27T21:51:44.987Z 28 560-579 184 183 184 331 329 331 F T F <---
199442024-02-27T21:51:44.987Z 29 580-599 178 178 178 322 322 322 T F T <---
199452024-02-27T21:51:44.987ZMax gen: 184, Max flush: 331
199462024-02-27T21:51:44.987ZError: Difference in extent metadata found!
199472024-02-27T21:51:44.988ZA Difference in extent metadata is expected here
199482024-02-27T21:51:44.988ZOn loop 92
199492024-02-27T21:51:44.989Z
199502024-02-27T21:51:44.989ZVerifying data now
199512024-02-27T21:51:49.620ZNo change after verify
199522024-02-27T21:51:49.620ZLoop: 92 Downstairs dump after verify (and repair):
199532024-02-27T21:51:49.632ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199542024-02-27T21:51:49.632Z 0 000-019 184 184 184 331 331 331 F F F
199552024-02-27T21:51:49.646Z 1 020-039 184 184 184 331 331 331 F F F
199562024-02-27T21:51:49.647Z 2 040-059 179 179 179 323 323 323 F F F
199572024-02-27T21:51:49.647Z 3 060-079 185 185 185 332 332 332 F F F
199582024-02-27T21:51:49.647Z 4 080-099 170 170 170 309 309 309 F F F
199592024-02-27T21:51:49.647Z 5 100-119 182 182 182 328 328 328 F F F
199602024-02-27T21:51:49.648Z 6 120-139 182 182 182 328 328 328 F F F
199612024-02-27T21:51:49.648Z 7 140-159 180 180 180 325 325 325 F F F
199622024-02-27T21:51:49.648Z 8 160-179 182 182 182 328 328 328 F F F
199632024-02-27T21:51:49.648Z 9 180-199 182 182 182 328 328 328 F F F
199642024-02-27T21:51:49.649Z 10 200-219 185 185 185 332 332 332 F F F
199652024-02-27T21:51:49.649Z 11 220-239 185 185 185 332 332 332 F F F
199662024-02-27T21:51:49.649Z 12 240-259 166 166 166 302 302 302 F F F
199672024-02-27T21:51:49.649Z 13 260-279 185 185 185 332 332 332 F F F
199682024-02-27T21:51:49.649Z 14 280-299 185 185 185 332 332 332 F F F
199692024-02-27T21:51:49.649Z 15 300-319 180 180 180 325 325 325 F F F
199702024-02-27T21:51:49.650Z 16 320-339 182 182 182 328 328 328 F F F
199712024-02-27T21:51:49.650Z 17 340-359 175 175 175 315 315 315 F F F
199722024-02-27T21:51:49.651Z 18 360-379 185 185 185 332 332 332 F F F
199732024-02-27T21:51:49.651Z 19 380-399 178 178 178 322 322 322 F F F
199742024-02-27T21:51:49.651Z 20 400-419 185 185 185 332 332 332 F F F
199752024-02-27T21:51:49.651Z 21 420-439 182 182 182 328 328 328 F F F
199762024-02-27T21:51:49.651Z 22 440-459 185 185 185 332 332 332 F F F
199772024-02-27T21:51:49.651Z 23 460-479 182 182 182 328 328 328 F F F
199782024-02-27T21:51:49.652Z 24 480-499 183 183 183 329 329 329 F F F
199792024-02-27T21:51:49.652Z 25 500-519 185 185 185 332 332 332 F F F
199802024-02-27T21:51:49.652Z 26 520-539 182 182 182 328 328 328 F F F
199812024-02-27T21:51:49.653Z 27 540-559 184 184 184 330 330 330 F F F
199822024-02-27T21:51:49.653Z 28 560-579 184 184 184 331 331 331 F F F
199832024-02-27T21:51:49.653Z 29 580-599 185 185 185 332 332 332 F F F
199842024-02-27T21:51:49.653ZMax gen: 185, Max flush: 332
199852024-02-27T21:51:49.653Z
199862024-02-27T21:51:49.654ZBegin loop 93
199872024-02-27T21:51:49.654ZDownstairs to restart: 2
199882024-02-27T21:51:49.674Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.003325367Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894}
199892024-02-27T21:51:49.675Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time"Crucible runtime is spawned
199902024-02-27T21:51:49.675Z:"2024-02-27T21:51:48.003643929Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894}
199912024-02-27T21:51:49.675Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.003690933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894}
199922024-02-27T21:51:49.675Z{"msg":"Crucible 6ad90105-4ef5-4775-a593-7ae279dd1de4 has session id: ad92a567-4267-47ec-87a2-f37ebeca4dfe","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.00372057Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
199932024-02-27T21:51:49.676Z{"msg":"Upstairs opts: Upstairs UUID: 6ad90105-4ef5-4775-a593-7ae279dd1de4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.003787012Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
199942024-02-27T21:51:49.676Z{"msg":"Crucible stats registered with UUID: 6ad90105-4ef5-4775-a593-7ae279dd1de4","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.003820028Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
199952024-02-27T21:51:49.676Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.003848905Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894}
199962024-02-27T21:51:49.677Z{"msg":"6ad90105-4ef5-4775-a593-7ae279dd1de4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.00388868Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
199972024-02-27T21:51:49.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.003916457Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
199982024-02-27T21:51:49.677Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.003964491Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
199992024-02-27T21:51:49.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.003997357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200002024-02-27T21:51:49.677Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.004031604Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200012024-02-27T21:51:49.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.004049791Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200022024-02-27T21:51:49.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.004067949Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200032024-02-27T21:51:49.678Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.004085097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200042024-02-27T21:51:49.678Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.004119123Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200052024-02-27T21:51:49.679Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.004197734Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200062024-02-27T21:51:49.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.004288233Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200072024-02-27T21:51:49.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.004360205Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200082024-02-27T21:51:49.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.00439975Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200092024-02-27T21:51:49.680Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.005091428Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200102024-02-27T21:51:49.680Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.005119465Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200112024-02-27T21:51:49.680Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.005155801Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200122024-02-27T21:51:49.681Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.005190697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200132024-02-27T21:51:49.681Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.005220773Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200142024-02-27T21:51:49.681Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.00524789Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200152024-02-27T21:51:49.682Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.00575728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200162024-02-27T21:51:49.682Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.005906243Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200172024-02-27T21:51:49.682Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.005950307Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200182024-02-27T21:51:49.682Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006216566Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200192024-02-27T21:51:49.683Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006258341Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200202024-02-27T21:51:49.683Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006287188Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200212024-02-27T21:51:49.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006386716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200222024-02-27T21:51:49.683Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006451139Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200232024-02-27T21:51:49.683Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006512381Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200242024-02-27T21:51:49.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006533379Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200252024-02-27T21:51:49.684Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006549317Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200262024-02-27T21:51:49.684Z{"msg":"[0]R flush_numbers[0..12]: [331, 331, 323, 332, 309, 328, 328, 325, 328, 328, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006564555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200272024-02-27T21:51:49.685Z{"msg":"[0]R generation[0..12]: [184, 184, 179, 185, 170, 182, 182, 180, 182, 182, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006587082Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200282024-02-27T21:51:49.685Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006601981Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200292024-02-27T21:51:49.685Z{"msg":"[1]R flush_numbers[0..12]: [331, 331, 323, 332, 309, 328, 328, 325, 328, 328, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006622988Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200302024-02-27T21:51:49.685Z{"msg":"[1]R generation[0..12]: [184, 184, 179, 185, 170, 182, 182, 180, 182, 182, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006637457Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200312024-02-27T21:51:49.686Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006658274Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200322024-02-27T21:51:49.686Z{"msg":"[2]R flush_numbers[0..12]: [331, 331, 323, 332, 309, 328, 328, 325, 328, 328, 332, 332]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006672762Z","hostname":"Wait for a query_work_queue command to finish before sending IO
200332024-02-27T21:51:49.686Zip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200342024-02-27T21:51:49.686Z{"msg":"[2]R generation[0..12]: [184, 184, 179, 185, 170, 182, 182, 180, 182, 182, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.00669737Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200352024-02-27T21:51:49.686Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006714357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200362024-02-27T21:51:49.686Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006740924Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200372024-02-27T21:51:49.686Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006755952Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200382024-02-27T21:51:49.686Z{"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.00677615Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200392024-02-27T21:51:49.686Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006789808Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"":"downstairs","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200402024-02-27T21:51:49.687Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006810246Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200412024-02-27T21:51:49.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006823275Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"0","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200422024-02-27T21:51:49.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006843932Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"1","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200432024-02-27T21:51:49.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006857631Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"client":"2","":"downstairs","session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200442024-02-27T21:51:49.687Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006889327Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200452024-02-27T21:51:49.687Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006910944Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200462024-02-27T21:51:49.687Z{"msg":"6ad90105-4ef5-4775-a593-7ae279dd1de4 is now active with session: ad92a567-4267-47ec-87a2-f37ebeca4dfe","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006942801Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200472024-02-27T21:51:49.687Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006966908Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894,"session_id":"ad92a567-4267-47ec-87a2-f37ebeca4dfe"}
200482024-02-27T21:51:49.687Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:48.006986345Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894}
200492024-02-27T21:51:49.688ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
200502024-02-27T21:51:49.688ZLoading write count information from file "/tmp/test_repair_verify.data"
200512024-02-27T21:51:49.688ZRun Repair workload
200522024-02-27T21:51:49.688Z01/30 Flush
200532024-02-27T21:51:49.688Z02/30 Write block 500 len 32768 data: 7 7 9 9 10 11 12 11
200542024-02-27T21:51:49.688Z03/30 Read block 555 len 28672
200552024-02-27T21:51:49.689Z04/30 Write block 78 len 36864 data: 12 11 11 11 9 8 10 11 12
200562024-02-27T21:51:49.689Z05/30 Read block 60 len 40960
200572024-02-27T21:51:50.137Z06/30 Read block 536 len 24576
200582024-02-27T21:51:50.137Z07/30 Read block 53 len 36864
200592024-02-27T21:51:50.137Z08/30 Flush
200602024-02-27T21:51:50.138Z09/30 Write block 567 len 12288 data: 10 12 12
200612024-02-27T21:51:50.138Z10/30 Write block 369 len 32768 data: 9 10 10 8 9 11 11 11
200622024-02-27T21:51:50.138Z11/30 Write block 329 len 4096 data: 16
200632024-02-27T21:51:50.138Z12/30 Write block 573 len 24576 data: 15 13 13 14 14 15
200642024-02-27T21:51:50.138Z13/30 Read block 380 len 16384
200652024-02-27T21:51:50.139Z14/30 Read block 408 len 28672
200662024-02-27T21:51:50.139Z15/30 Read block 85 len 36864
200672024-02-27T21:51:50.139Z16/30 Read block 506 len 24576
200682024-02-27T21:51:50.140Z17/30 Read block 111 len 40960
200692024-02-27T21:51:50.140Z18/30 Write block 114 len 4096 data: 17
200702024-02-27T21:51:50.140Z19/30 Write block 373 len 20480 data: 10 12 12 12 13
200712024-02-27T21:51:50.140Z20/30 Write block 505 len 20480 data: 12 13 12 8 12
200722024-02-27T21:51:50.141Z21/30 Read block 50 len 24576
200732024-02-27T21:51:50.141Z22/30 Read block 461 len 12288
200742024-02-27T21:51:50.141Z23/30 Write block 238 len 12288 data: 13 13 8
200752024-02-27T21:51:50.141Z24/30 Read block 45 len 40960
200762024-02-27T21:51:50.141Z25/30 Read block 55 len 16384
200772024-02-27T21:51:50.142Z26/30 Flush
200782024-02-27T21:51:50.142Z27/30 Write block 130 len 24576 data: 12 10 7 9 9 11
200792024-02-27T21:51:50.142Z28/30 Write block 430 len 20480 data: 15 15 15 15 16
200802024-02-27T21:51:50.142Z29/30 Read block 100 len 36864
200812024-02-27T21:51:50.143Z30/30 Write block 462 len 28672 data: 12 11 11 12 13 14 14
200822024-02-27T21:51:50.143Z----------------------------------------------------------------
200832024-02-27T21:51:50.143Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:29
200842024-02-27T21:51:50.144ZStates: Active Active Active
200852024-02-27T21:51:50.144ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200862024-02-27T21:51:50.144Z 2 Acked 1001 Write 8 Done Done Done false
200872024-02-27T21:51:50.144Z 3 Acked 1002 Read 7 Done Done Done false
200882024-02-27T21:51:50.144Z 4 Acked 1003 Write 9 Done Done Sent false
200892024-02-27T21:51:50.144Z 5 Acked 1004 Read 10 Done Done Sent false
200902024-02-27T21:51:50.144Z 6 Acked 1005 Read 6 Done Done Sent false
200912024-02-27T21:51:50.145Z 7 Acked 1006 Read 9 Done Done Sent false
200922024-02-27T21:51:50.145Z 8 Acked 1007 Flush 0 Done Done Sent false
200932024-02-27T21:51:50.145Z 9 Acked 1008 Write 3 Done Done Sent false
200942024-02-27T21:51:50.146Z 10 Acked 1009 Write 8 Done Done Sent false
200952024-02-27T21:51:50.146Z 11 Acked 1010 Write 1 Done Done Sent false
200962024-02-27T21:51:50.146Z 12 Acked 1011 Write 6 Done Done Sent false
200972024-02-27T21:51:50.146Z 13 Acked 1012 Read 4 Done Done Sent false
200982024-02-27T21:51:50.146Z 14 Acked 1013 Read 7 Done Done Sent false
200992024-02-27T21:51:50.146Z 15 Acked 1014 Read 9 Done Done Sent false
201002024-02-27T21:51:50.147Z 16 Acked 1015 Read 6 Done Done Sent false
201012024-02-27T21:51:50.147Z 17 Acked 1016 Read 10 Done Done Sent false
201022024-02-27T21:51:50.147Z 18 Acked 1017 Write 1 Done Done Sent false
201032024-02-27T21:51:50.147Z 19 Acked 1018 Write 5 Done Done Sent false
201042024-02-27T21:51:50.147Z 20 Acked 1019 Write 5 Done Done Sent false
201052024-02-27T21:51:50.147Z 21 Acked 1020 Read 6 Done Done Sent false
201062024-02-27T21:51:50.147Z 22 Acked 1021 Read 3 Done Done Sent false
201072024-02-27T21:51:50.148Z 23 Acked 1022 Write 3 Done Done Sent false
201082024-02-27T21:51:50.148Z 24 Acked 1023 Read 10 Done Done Sent false
201092024-02-27T21:51:50.148Z 25 Acked 1024 Read 4 Done Done Sent false
201102024-02-27T21:51:50.148Z 26 Acked 1025 Flush 0 Done Done Sent false
201112024-02-27T21:51:50.148Z 27 Acked 1026 Write 6 Done Done Sent false
201122024-02-27T21:51:50.148Z 28 Acked 1027 Write 5 Done Done Sent false
201132024-02-27T21:51:50.149Z 29 Acked 1028 Read 9 Done Done Sent false
201142024-02-27T21:51:50.149Z 30 Acked 1029 Write 7 Sent Sent Sent false
201152024-02-27T21:51:50.149Z STATES DS:0 DS:1 DS:2 TOTAL
201162024-02-27T21:51:50.149Z New 0 0 0 0
201172024-02-27T21:51:50.149Z Sent 1 1 27 29
201182024-02-27T21:51:50.149Z Done 28 28 2 58
201192024-02-27T21:51:50.150Z Skipped 0 0 0 0
201202024-02-27T21:51:50.150Z Error 0 0 0 0
201212024-02-27T21:51:50.150ZLast Flush: 1025 1025 1000
201222024-02-27T21:51:50.150ZDownstairs last five completed: 1000
201232024-02-27T21:51:50.151ZUpstairs last five completed: 30 29 28 27 26
201242024-02-27T21:51:50.151Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:48.03903766Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1894}
201252024-02-27T21:51:50.151ZWrote out file "/tmp/test_repair_verify.data"
201262024-02-27T21:51:50.151Z
201272024-02-27T21:51:50.151ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
201282024-02-27T21:51:50.151ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201292024-02-27T21:51:50.151Z 0 000-019 184 184 184 331 331 331 F F F
201302024-02-27T21:51:50.152Z 1 020-039 184 184 184 331 331 331 F F F
201312024-02-27T21:51:50.152Z 2 040-059 179 179 179 323 323 323 F F F
201322024-02-27T21:51:50.152Z 3 060-079 186 186 185 334 334 332 F F F <---
201332024-02-27T21:51:50.152Z 4 080-099 186 186 170 334 334 309 F F F <---
201342024-02-27T21:51:50.152Z 5 100-119 186 186 182 335 335 328 F F F <---
201352024-02-27T21:51:50.153Z 6 120-139 182 182 182 328 328 328 T T F <---
201362024-02-27T21:51:50.153Z 7 140-159 180 180 180 325 325 325 F F F
201372024-02-27T21:51:50.153Z 8 160-179 182 182 182 328 328 328 F F F
201382024-02-27T21:51:50.153Z 9 180-199 182 182 182 328 328 328 F F F
201392024-02-27T21:51:50.154Z 10 200-219 185 185 185 332 332 332 F F F
201402024-02-27T21:51:50.154Z 11 220-239 186 186 185 335 335 332 F F F <---
201412024-02-27T21:51:50.154Z 12 240-259 186 186 166 335 335 302 F F F <---
201422024-02-27T21:51:50.154Z 13 260-279 185 185 185 332 332 332 F F F
201432024-02-27T21:51:50.155Z 14 280-299 185 185 185 332 332 332 F F F
201442024-02-27T21:51:50.155Z 15 300-319 180 180 180 325 325 325 F F F
201452024-02-27T21:51:50.155Z 16 320-339 186 186 182 335 335 328 F F F <---
201462024-02-27T21:51:50.155Z 17 340-359 175 175 175 315 315 315 F F F
201472024-02-27T21:51:50.155Z 18 360-379 186 186 185 335 335 332 F F F <---
201482024-02-27T21:51:50.155Z 19 380-399 178 178 178 322 322 322 F F F
201492024-02-27T21:51:50.155Z 20 400-419 185 185 185 332 332 332 F F F
201502024-02-27T21:51:50.155Z 21 420-439 182 182 182 328 328 328 T T F <---
201512024-02-27T21:51:50.155Z 22 440-459 185 185 185 332 332 332 F F F
201522024-02-27T21:51:50.156Z 23 460-479 182 182 182 328 328 328 T T F <---
201532024-02-27T21:51:50.156Z 24 480-499 183 183 183 329 329 329 F F F
201542024-02-27T21:51:50.156Z 25 500-519 186 186 185 335 335 332 F F T <---
201552024-02-27T21:51:50.156Z 26 520-539 182 182 182 328 328 328 F F F
201562024-02-27T21:51:50.156Z 27 540-559 184 184 184 330 330 330 F F F
201572024-02-27T21:51:50.176Z 28 560-579 186 186 184 335 335 331 F F F <---
201582024-02-27T21:51:50.177Z 29 580-599 185 185 185 332 332 332 F F F
201592024-02-27T21:51:50.177ZMax gen: 186, Max flush: 335
201602024-02-27T21:51:50.177ZError: Difference in extent metadata found!
201612024-02-27T21:51:50.177ZA Difference in extent metadata is expected here
201622024-02-27T21:51:50.177ZOn loop 93
201632024-02-27T21:51:50.177Z
201642024-02-27T21:51:50.178ZVerifying data now
201652024-02-27T21:51:54.325ZNo change after verify
201662024-02-27T21:51:54.325ZLoop: 93 Downstairs dump after verify (and repair):
201672024-02-27T21:51:54.339ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201682024-02-27T21:51:54.339Z 0 000-019 184 184 184 331 331 331 F F F
201692024-02-27T21:51:54.339Z 1 020-039 184 184 184 331 331 331 F F F
201702024-02-27T21:51:54.339Z 2 040-059 179 179 179 323 323 323 F F F
201712024-02-27T21:51:54.339Z 3 060-079 186 186 186 334 334 334 F F F
201722024-02-27T21:51:54.339Z 4 080-099 186 186 186 334 334 334 F F F
201732024-02-27T21:51:54.339Z 5 100-119 186 186 186 335 335 335 F F F
201742024-02-27T21:51:54.343Z 6 120-139 187 187 187 336 336 336 F F F
201752024-02-27T21:51:54.343Z 7 140-159 180 180 180 325 325 325 F F F
201762024-02-27T21:51:54.343Z 8 160-179 182 182 182 328 328 328 F F F
201772024-02-27T21:51:54.343Z 9 180-199 182 182 182 328 328 328 F F F
201782024-02-27T21:51:54.343Z 10 200-219 185 185 185 332 332 332 F F F
201792024-02-27T21:51:54.343Z 11 220-239 186 186 186 335 335 335 F F F
201802024-02-27T21:51:54.343Z 12 240-259 186 186 186 335 335 335 F F F
201812024-02-27T21:51:54.343Z 13 260-279 185 185 185 332 332 332 F F F
201822024-02-27T21:51:54.343Z 14 280-299 185 185 185 332 332 332 F F F
201832024-02-27T21:51:54.344Z 15 300-319 180 180 180 325 325 325 F F F
201842024-02-27T21:51:54.344Z 16 320-339 186 186 186 335 335 335 F F F
201852024-02-27T21:51:54.344Z 17 340-359 175 175 175 315 315 315 F F F
201862024-02-27T21:51:54.344Z 18 360-379 186 186 186 335 335 335 F F F
201872024-02-27T21:51:54.345Z 19 380-399 178 178 178 322 322 322 F F F
201882024-02-27T21:51:54.346Z 20 400-419 185 185 185 332 332 332 F F F
201892024-02-27T21:51:54.346Z 21 420-439 187 187 187 336 336 336 F F F
201902024-02-27T21:51:54.346Z 22 440-459 185 185 185 332 332 332 F F F
201912024-02-27T21:51:54.346Z 23 460-479 187 187 187 336 336 336 F F F
201922024-02-27T21:51:54.346Z 24 480-499 183 183 183 329 329 329 F F F
201932024-02-27T21:51:54.346Z 25 500-519 186 186 186 335 335 335 F F F
201942024-02-27T21:51:54.346Z 26 520-539 182 182 182 328 328 328 F F F
201952024-02-27T21:51:54.346Z 27 540-559 184 184 184 330 330 330 F F F
201962024-02-27T21:51:54.347Z 28 560-579 186 186 186 335 335 335 F F F
201972024-02-27T21:51:54.347Z 29 580-599 185 185 185 332 332 332 F F F
201982024-02-27T21:51:54.347ZMax gen: 187, Max flush: 336
201992024-02-27T21:51:54.347Z
202002024-02-27T21:51:54.347ZBegin loop 94
202012024-02-27T21:51:54.347ZDownstairs to restart: 2
202022024-02-27T21:51:54.775Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.712732205Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902}
202032024-02-27T21:51:54.808Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.712986125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902}
202042024-02-27T21:51:54.808Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713015682Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902}
202052024-02-27T21:51:54.808Z{"msg":"Crucible c6ce3bc0-59cc-4a4a-81f5-ed9cb082a863 has session id: 2eeefeca-5ae4-4cd9-9801-5ed8657904c7","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713039169Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202062024-02-27T21:51:54.809Z{"msg":"Upstairs opts: Upstairs UUID: c6ce3bc0-59cc-4a4a-81f5-ed9cb082a863, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false"Crucible runtime is spawned
202072024-02-27T21:51:54.809Z,"v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713081634Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202082024-02-27T21:51:54.809Z{"msg":"Crucible stats registered with UUID: c6ce3bc0-59cc-4a4a-81f5-ed9cb082a863","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.71311552Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202092024-02-27T21:51:54.810Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713140357Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902}
202102024-02-27T21:51:54.810Z{"msg":"c6ce3bc0-59cc-4a4a-81f5-ed9cb082a863 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713311017Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202112024-02-27T21:51:54.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713358531Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202122024-02-27T21:51:54.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713394067Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202132024-02-27T21:51:54.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713432163Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202142024-02-27T21:51:54.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713467809Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202152024-02-27T21:51:54.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713504204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202162024-02-27T21:51:54.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.71354205Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202172024-02-27T21:51:54.813Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713569317Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202182024-02-27T21:51:54.813Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713653716Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202192024-02-27T21:51:54.813Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713702381Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202202024-02-27T21:51:54.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713740306Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202212024-02-27T21:51:54.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713776592Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202222024-02-27T21:51:54.814Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.713804059Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202232024-02-27T21:51:54.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.714392309Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202242024-02-27T21:51:54.815Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.714427765Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202252024-02-27T21:51:54.815Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.714443903Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202262024-02-27T21:51:54.815Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.714457982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202272024-02-27T21:51:54.816Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.714479709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202282024-02-27T21:51:54.816Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.714494268Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202292024-02-27T21:51:54.816Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715006227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202302024-02-27T21:51:54.816Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715043633Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202312024-02-27T21:51:54.817Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.71506653Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202322024-02-27T21:51:54.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715537475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202332024-02-27T21:51:54.817Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715582839Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202342024-02-27T21:51:54.817Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715644972Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202352024-02-27T21:51:54.818Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715699895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202362024-02-27T21:51:54.818Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715737741Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202372024-02-27T21:51:54.818Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715754999Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202382024-02-27T21:51:54.818Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715775747Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202392024-02-27T21:51:54.819Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715790275Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202402024-02-27T21:51:54.819Z{"msg":"[0]R flush_numbers[0..12]: [331, 331, 323, 334, 334, 335, 336, 325, 328, 328, 332, 335]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715810993Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202412024-02-27T21:51:54.819Z{"msg":"[0]R generation[0..12]: [184, 184, 179, 186, 186, 186, 187, 180, 182, 182, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715825651Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202422024-02-27T21:51:54.819Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715853957Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202432024-02-27T21:51:54.820Z{"msg":"[1]R flush_numbers[0..12]: [331, 331, 323, 334, 334, 335, 336, 325, 328, 328, 332, 335]","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO
202442024-02-27T21:51:54.820Z"2024-02-27T21:51:52.715870355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202452024-02-27T21:51:54.820Z{"msg":"[1]R generation[0..12]: [184, 184, 179, 186, 186, 186, 187, 180, 182, 182, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.71591542Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202462024-02-27T21:51:54.820Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715951806Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202472024-02-27T21:51:54.820Z{"msg":"[2]R flush_numbers[0..12]: [331, 331, 323, 334, 334, 335, 336, 325, 328, 328, 332, 335]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.715979013Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202482024-02-27T21:51:54.820Z{"msg":"[2]R generation[0..12]: [184, 184, 179, 186, 186, 186, 187, 180, 182, 182, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716011929Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202492024-02-27T21:51:54.820Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716038636Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202502024-02-27T21:51:54.820Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716069892Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202512024-02-27T21:51:54.821Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716094379Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202522024-02-27T21:51:54.821Z{"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716120836Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202532024-02-27T21:51:54.821Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716144353Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"":"downstairs","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202542024-02-27T21:51:54.821Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.71617144Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202552024-02-27T21:51:54.821Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716193627Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"0","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202562024-02-27T21:51:54.821Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716220424Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"1","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202572024-02-27T21:51:54.821Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716244282Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"client":"2","":"downstairs","session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202582024-02-27T21:51:54.822Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716271128Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202592024-02-27T21:51:54.823Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716293386Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202602024-02-27T21:51:54.824Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202612024-02-27T21:51:54.826Zc6ce3bc0-59cc-4a4a-81f5-ed9cb082a863 is now active with session: 2eeefeca-5ae4-4cd9-9801-5ed8657904c7","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716330071Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202622024-02-27T21:51:54.830Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716360778Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902,"session_id":"2eeefeca-5ae4-4cd9-9801-5ed8657904c7"}
202632024-02-27T21:51:54.832Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:52.716384995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902}
202642024-02-27T21:51:54.832ZLoading write count information from file "/tmp/test_repair_verify.data"
202652024-02-27T21:51:54.832ZRun Repair workload
202662024-02-27T21:51:54.832Z01/30 Write block 312 len 4096 data: 18
202672024-02-27T21:51:54.832Z02/30 Read block 146 len 32768
202682024-02-27T21:51:54.833Z03/30 Read block 104 len 40960
202692024-02-27T21:51:54.833Z04/30 Read block 268 len 28672
202702024-02-27T21:51:54.833Z05/30 Write block 102 len 40960 data: 14 13 11 14 17 20 21 18 18 22
202712024-02-27T21:51:54.834Z06/30 Read block 128 len 20480
202722024-02-27T21:51:54.834Z07/30 Write block 498 len 36864 data: 10 9 8 8 10 10 11 13 14
202732024-02-27T21:51:54.834Z08/30 Write block 571 len 40960 data: 11 14 16 14 14 15 15 16 11 11
202742024-02-27T21:51:54.834Z09/30 Read block 171 len 16384
202752024-02-27T21:51:54.835Z10/30 Write block 508 len 12288 data: 9 13 14
202762024-02-27T21:51:54.835Z11/30 Read block 450 len 4096
202772024-02-27T21:51:54.835Z12/30 Write block 306 len 40960 data: 14 16 15 16 18 19 19 16 17 13
202782024-02-27T21:51:54.835Z13/30 Read block 57 len 8192
202792024-02-27T21:51:54.835Z14/30 Read block 168 len 36864
202802024-02-27T21:51:54.836Z15/30 Read block 214 len 4096
202812024-02-27T21:51:54.836Z16/30 Read block 177 len 40960
202822024-02-27T21:51:54.836Z17/30 Write block 123 len 36864 data: 18 19 14 12 12 15 14 13 11
202832024-02-27T21:51:54.836Z18/30 Write block 18 len 8192 data: 18 17
202842024-02-27T21:51:54.836Z19/30 Read block 525 len 12288
202852024-02-27T21:51:54.836Z20/30 Write block 569 len 16384 data: 13 10 12 15
202862024-02-27T21:51:54.837Z21/30 Read block 287 len 28672
202872024-02-27T21:51:54.837Z22/30 Write block 492 len 12288 data: 14 10 13
202882024-02-27T21:51:54.837Z23/30 Write block 527 len 40960 data: 16 14 13 11 11 9 11 14 13 12
202892024-02-27T21:51:54.837Z24/30 Read block 567 len 8192
202902024-02-27T21:51:54.837Z25/30 Write block 455 len 8192 data: 6 4
202912024-02-27T21:51:54.838Z26/30 Write block 372 len 36864 data: 9 11 13 13 13 14 8 11 12
202922024-02-27T21:51:54.838Z27/30 Write block 566 len 8192 data: 10 11
202932024-02-27T21:51:54.838Z28/30 Read block 392 len 8192
202942024-02-27T21:51:54.838Z29/30 Read block 350 len 40960
202952024-02-27T21:51:54.838Z30/30 Read block 540 len 28672
202962024-02-27T21:51:54.838Z----------------------------------------------------------------
202972024-02-27T21:51:54.838Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
202982024-02-27T21:51:54.839ZStates: Active Active Active
202992024-02-27T21:51:54.839ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
203002024-02-27T21:51:54.839Z 1 Acked 1000 Write 1 Done Done Done false
203012024-02-27T21:51:54.839Z 2 Acked 1001 Read 8 Done Done Done false
203022024-02-27T21:51:54.839Z 3 Acked 1002 Read 10 Done Done Done false
203032024-02-27T21:51:54.866Z 4 Acked 1003 Read 7 Done Done Done false
203042024-02-27T21:51:54.866Z 5 Acked 1004 Write 10 Done Done Done false
203052024-02-27T21:51:54.867Z 6 Acked 1005 Read 5 Done Done Done false
203062024-02-27T21:51:54.867Z 7 Acked 1006 Write 9 Done Done Sent false
203072024-02-27T21:51:54.867Z 8 Acked 1007 Write 10 Done Done Sent false
203082024-02-27T21:51:54.867Z 9 Acked 1008 Read 4 Done Done Sent false
203092024-02-27T21:51:54.867Z 10 Acked 1009 Write 3 Done Done Sent false
203102024-02-27T21:51:54.867Z 11 Acked 1010 Read 1 Done Done Sent false
203112024-02-27T21:51:54.868Z 12 Acked 1011 Write 10 Done Done Sent false
203122024-02-27T21:51:54.868Z 13 Acked 1012 Read 2 Done Done Sent false
203132024-02-27T21:51:54.868Z 14 Acked 1013 Read 9 Done Done Sent false
203142024-02-27T21:51:54.868Z 15 Acked 1014 Read 1 Done Done Sent false
203152024-02-27T21:51:54.868Z 16 Acked 1015 Read 10 Done Done Sent false
203162024-02-27T21:51:54.869Z 17 Acked 1016 Write 9 Done Done Sent false
203172024-02-27T21:51:54.869Z 18 Acked 1017 Write 2 Done Done Sent false
203182024-02-27T21:51:54.869Z 19 Acked 1018 Read 3 Done Done Sent false
203192024-02-27T21:51:54.869Z 20 Acked 1019 Write 4 Done Done Sent false
203202024-02-27T21:51:54.869Z 21 Acked 1020 Read 7 Done Done Sent false
203212024-02-27T21:51:54.870Z 22 Acked 1021 Write 3 Done Done Sent false
203222024-02-27T21:51:54.870Z 23 Acked 1022 Write 10 Done Done Sent false
203232024-02-27T21:51:54.870Z 24 Acked 1023 Read 2 Done Done Sent false
203242024-02-27T21:51:54.870Z 25 Acked 1024 Write 2 Done Done Sent false
203252024-02-27T21:51:54.870Z 26 Acked 1025 Write 9 Done Done Sent false
203262024-02-27T21:51:54.870Z 27 Acked 1026 Write 2 Done Done Sent false
203272024-02-27T21:51:54.870Z 28 Acked 1027 Read 2 Done Done Sent false
203282024-02-27T21:51:54.871Z 29 Acked 1028 Read 10 Done Done Sent false
203292024-02-27T21:51:54.871Z 30 Acked 1029 Read 7 Done Done Sent false
203302024-02-27T21:51:54.871Z STATES DS:0 DS:1 DS:2 TOTAL
203312024-02-27T21:51:54.871Z New 0 0 0 0
203322024-02-27T21:51:54.871Z Sent 0 0 24 24
203332024-02-27T21:51:54.871Z Done 30 30 6 66
203342024-02-27T21:51:54.871Z Skipped 0 0 0 0
203352024-02-27T21:51:54.871Z Error 0 0 0 0
203362024-02-27T21:51:54.872ZLast Flush: 0 0 0
203372024-02-27T21:51:54.872ZDownstairs last five completed:
203382024-02-27T21:51:54.873ZUpstairs last five completed: 30 29 28 27 26
203392024-02-27T21:51:54.873Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:52.743787497Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1902}
203402024-02-27T21:51:54.873ZWrote out file "/tmp/test_repair_verify.data"
203412024-02-27T21:51:54.873Z
203422024-02-27T21:51:54.873ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
203432024-02-27T21:51:54.873ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203442024-02-27T21:51:54.873Z 0 000-019 184 184 184 331 331 331 T T F <---
203452024-02-27T21:51:54.874Z 1 020-039 184 184 184 331 331 331 F F F
203462024-02-27T21:51:54.874Z 2 040-059 179 179 179 323 323 323 F F F
203472024-02-27T21:51:54.874Z 3 060-079 186 186 186 334 334 334 F F F
203482024-02-27T21:51:54.874Z 4 080-099 186 186 186 334 334 334 F F F
203492024-02-27T21:51:54.874Z 5 100-119 186 186 186 335 335 335 T T T
203502024-02-27T21:51:54.874Z 6 120-139 187 187 187 336 336 336 T T F <---
203512024-02-27T21:51:54.875Z 7 140-159 180 180 180 325 325 325 F F F
203522024-02-27T21:51:54.875Z 8 160-179 182 182 182 328 328 328 F F F
203532024-02-27T21:51:54.875Z 9 180-199 182 182 182 328 328 328 F F F
203542024-02-27T21:51:54.875Z 10 200-219 185 185 185 332 332 332 F F F
203552024-02-27T21:51:54.876Z 11 220-239 186 186 186 335 335 335 F F F
203562024-02-27T21:51:54.876Z 12 240-259 186 186 186 335 335 335 F F F
203572024-02-27T21:51:54.876Z 13 260-279 185 185 185 332 332 332 F F F
203582024-02-27T21:51:54.876Z 14 280-299 185 185 185 332 332 332 F F F
203592024-02-27T21:51:54.876Z 15 300-319 180 180 180 325 325 325 T T T
203602024-02-27T21:51:54.876Z 16 320-339 186 186 186 335 335 335 F F F
203612024-02-27T21:51:54.876Z 17 340-359 175 175 175 315 315 315 F F F
203622024-02-27T21:51:54.877Z 18 360-379 186 186 186 335 335 335 T T F <---
203632024-02-27T21:51:54.877Z 19 380-399 178 178 178 322 322 322 T T F <---
203642024-02-27T21:51:54.877Z 20 400-419 185 185 185 332 332 332 F F F
203652024-02-27T21:51:54.877Z 21 420-439 187 187 187 336 336 336 F F F
203662024-02-27T21:51:54.877Z 22 440-459 185 185 185 332 332 332 T T F <---
203672024-02-27T21:51:54.877Z 23 460-479 187 187 187 336 336 336 F F F
203682024-02-27T21:51:54.878Z 24 480-499 183 183 183 329 329 329 T T F <---
203692024-02-27T21:51:54.878Z 25 500-519 186 186 186 335 335 335 T T F <---
203702024-02-27T21:51:54.878Z 26 520-539 182 182 182 328 328 328 T T F <---
203712024-02-27T21:51:54.878Z 27 540-559 184 184 184 330 330 330 F F F
203722024-02-27T21:51:54.878Z 28 560-579 186 186 186 335 335 335 T T F <---
203732024-02-27T21:51:54.878Z 29 580-599 185 185 185 332 332 332 T T F <---
203742024-02-27T21:51:54.878ZMax gen: 187, Max flush: 336
203752024-02-27T21:51:54.878ZError: Difference in extent metadata found!
203762024-02-27T21:51:54.879ZA Difference in extent metadata is expected here
203772024-02-27T21:51:54.879ZOn loop 94
203782024-02-27T21:51:54.879Z
203792024-02-27T21:51:54.879ZVerifying data now
203802024-02-27T21:51:59.057ZNo change after verify
203812024-02-27T21:51:59.057ZLoop: 94 Downstairs dump after verify (and repair):
203822024-02-27T21:51:59.074ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203832024-02-27T21:51:59.075Z 0 000-019 188 188 188 337 337 337 F F F
203842024-02-27T21:51:59.075Z 1 020-039 184 184 184 331 331 331 F F F
203852024-02-27T21:51:59.075Z 2 040-059 179 179 179 323 323 323 F F F
203862024-02-27T21:51:59.075Z 3 060-079 186 186 186 334 334 334 F F F
203872024-02-27T21:51:59.075Z 4 080-099 186 186 186 334 334 334 F F F
203882024-02-27T21:51:59.075Z 5 100-119 188 188 188 337 337 337 F F F
203892024-02-27T21:51:59.076Z 6 120-139 188 188 188 337 337 337 F F F
203902024-02-27T21:51:59.076Z 7 140-159 180 180 180 325 325 325 F F F
203912024-02-27T21:51:59.076Z 8 160-179 182 182 182 328 328 328 F F F
203922024-02-27T21:51:59.076Z 9 180-199 182 182 182 328 328 328 F F F
203932024-02-27T21:51:59.076Z 10 200-219 185 185 185 332 332 332 F F F
203942024-02-27T21:51:59.076Z 11 220-239 186 186 186 335 335 335 F F F
203952024-02-27T21:51:59.076Z 12 240-259 186 186 186 335 335 335 F F F
203962024-02-27T21:51:59.076Z 13 260-279 185 185 185 332 332 332 F F F
203972024-02-27T21:51:59.183Z 14 280-299 185 185 185 332 332 332 F F F
203982024-02-27T21:51:59.184Z 15 300-319 188 188 188 337 337 337 F F F
203992024-02-27T21:51:59.184Z 16 320-339 186 186 186 335 335 335 F F F
204002024-02-27T21:51:59.184Z 17 340-359 175 175 175 315 315 315 F F F
204012024-02-27T21:51:59.184Z 18 360-379 188 188 188 337 337 337 F F F
204022024-02-27T21:51:59.184Z 19 380-399 188 188 188 337 337 337 F F F
204032024-02-27T21:51:59.185Z 20 400-419 185 185 185 332 332 332 F F F
204042024-02-27T21:51:59.185Z 21 420-439 187 187 187 336 336 336 F F F
204052024-02-27T21:51:59.185Z 22 440-459 188 188 188 337 337 337 F F F
204062024-02-27T21:51:59.186Z 23 460-479 187 187 187 336 336 336 F F F
204072024-02-27T21:51:59.186Z 24 480-499 188 188 188 337 337 337 F F F
204082024-02-27T21:51:59.186Z 25 500-519 188 188 188 337 337 337 F F F
204092024-02-27T21:51:59.186Z 26 520-539 188 188 188 337 337 337 F F F
204102024-02-27T21:51:59.186Z 27 540-559 184 184 184 330 330 330 F F F
204112024-02-27T21:51:59.186Z 28 560-579 188 188 188 337 337 337 F F F
204122024-02-27T21:51:59.186Z 29 580-599 188 188 188 337 337 337 F F F
204132024-02-27T21:51:59.186ZMax gen: 188, Max flush: 337
204142024-02-27T21:51:59.186Z
204152024-02-27T21:51:59.187ZBegin loop 95
204162024-02-27T21:51:59.187ZDownstairs to restart: 0
204172024-02-27T21:51:59.187Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.437596073Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910}
204182024-02-27T21:51:59.187Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.437863081Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910}
204192024-02-27T21:51:59.187Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.437880759Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910}
204202024-02-27T21:51:59.187Z{"msg":"Crucible a8563270-345a-496f-9c11-954d1a3c8120 has session id: 6ef47146-48ee-42c4-b050-8a73a9fa196b","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.437902087Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204212024-02-27T21:51:59.187Z{"msg":"Upstairs opts: Upstairs UUID: a8563270-345a-496f-9c11-954d1a3c8120, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.437935743Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204222024-02-27T21:51:59.188Z{"msg":"Crucible stats registered with UUID: a8563270-345a-496f-9c11-954d1a3c8120","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
204232024-02-27T21:51:59.188Z2024-02-27T21:51:57.437955801Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204242024-02-27T21:51:59.188Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438071417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910}
204252024-02-27T21:51:59.188Z{"msg":"a8563270-345a-496f-9c11-954d1a3c8120 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438190463Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204262024-02-27T21:51:59.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438233538Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204272024-02-27T21:51:59.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438261995Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204282024-02-27T21:51:59.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.43830299Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204292024-02-27T21:51:59.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438331846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204302024-02-27T21:51:59.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438358523Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204312024-02-27T21:51:59.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438394999Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204322024-02-27T21:51:59.189Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438422646Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204332024-02-27T21:51:59.190Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.43846751Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204342024-02-27T21:51:59.190Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438509236Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204352024-02-27T21:51:59.190Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438544331Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204362024-02-27T21:51:59.190Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438581887Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204372024-02-27T21:51:59.190Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.438618153Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204382024-02-27T21:51:59.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.439245329Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204392024-02-27T21:51:59.190Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.439309161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204402024-02-27T21:51:59.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.439343427Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204412024-02-27T21:51:59.191Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.439428857Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204422024-02-27T21:51:59.191Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.439472272Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204432024-02-27T21:51:59.191Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.439513437Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204442024-02-27T21:51:59.191Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.439819171Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204452024-02-27T21:51:59.191Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.439857087Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204462024-02-27T21:51:59.191Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440158931Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204472024-02-27T21:51:59.192Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440399323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204482024-02-27T21:51:59.192Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440448607Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204492024-02-27T21:51:59.192Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440468435Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204502024-02-27T21:51:59.192Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440483933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204512024-02-27T21:51:59.192Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440498361Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204522024-02-27T21:51:59.192Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440525358Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204532024-02-27T21:51:59.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440661772Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204542024-02-27T21:51:59.193Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440702557Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204552024-02-27T21:51:59.193Z{"msg":"[0]R flush_numbers[0..12]: [337, 331, 323, 334, 334, 337, 337, 325, 328, 328, 332, 335]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440720245Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204562024-02-27T21:51:59.193Z{"msg":"[0]R generation[0..12]: [188, 184, 179, 186, 186, 188, 188, 180, 182, 182, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440741933Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204572024-02-27T21:51:59.193Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440757431Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204582024-02-27T21:51:59.193Z{"msg":"[1]R flush_numbers[0..12]: [337, 331, 323, 334, 334, 337, 337, 325, 328, 328, 332, 335]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440778148Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204592024-02-27T21:51:59.193Z{"msg":"[1]R generation[0..12]: [188, 184, 179, 186, 186, 188, 188, 180, 182, 182, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440792456Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204602024-02-27T21:51:59.193Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440813284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204612024-02-27T21:51:59.194Z{"msg":"[2]R flush_numbers[0..12]: [337, 331, 323, 334, 334, 337, 337, 325, 328, 328, 332, 335]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440827832Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204622024-02-27T21:51:59.194Z{"msg":"[2]R generation[0..12]: [188, 184, 179, 186, 186, 188, 188, 180, 182, 182, 185, 186]","v":Wait for a query_work_queue command to finish before sending IO
204632024-02-27T21:51:59.194Z0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.44085126Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204642024-02-27T21:51:59.194Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440866438Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204652024-02-27T21:51:59.194Z{"msg":"Max found gen is 189","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440886365Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204662024-02-27T21:51:59.194Z{"msg":"Generation requested: 190 >= found:189","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440901294Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204672024-02-27T21:51:59.194Z{"msg":"Next flush: 338","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440921281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204682024-02-27T21:51:59.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.44093479Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"":"downstairs","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204692024-02-27T21:51:59.195Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440948318Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204702024-02-27T21:51:59.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440969136Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"0","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204712024-02-27T21:51:59.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.440982894Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"1","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204722024-02-27T21:51:59.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.441003302Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"client":"2","":"downstairs","session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204732024-02-27T21:51:59.195Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.44101687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204742024-02-27T21:51:59.195Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.441035378Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204752024-02-27T21:51:59.195Z{"msg":"a8563270-345a-496f-9c11-954d1a3c8120 is now active with session: 6ef47146-48ee-42c4-b050-8a73a9fa196b","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.441048846Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204762024-02-27T21:51:59.195Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.441068204Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910,"session_id":"6ef47146-48ee-42c4-b050-8a73a9fa196b"}
204772024-02-27T21:51:59.195Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:51:57.441080882Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910}
204782024-02-27T21:51:59.195ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204792024-02-27T21:51:59.195ZLoading write count information from file "/tmp/test_repair_verify.data"
204802024-02-27T21:51:59.195ZRun Repair workload
204812024-02-27T21:51:59.195Z01/30 Write block 238 len 24576 data: 14 14 9 9 7 10
204822024-02-27T21:51:59.195Z02/30 Write block 160 len 20480 data: 16 15 14 15 13
204832024-02-27T21:51:59.195Z03/30 Write block 178 len 28672 data: 16 15 16 19 20 17 17
204842024-02-27T21:51:59.196Z04/30 Read block 173 len 40960
204852024-02-27T21:51:59.196Z05/30 Read block 526 len 8192
204862024-02-27T21:51:59.196Z06/30 Flush
204872024-02-27T21:51:59.196Z07/30 Write block 348 len 32768 data: 21 20 19 18 15 13 12 18
204882024-02-27T21:51:59.196Z08/30 Read block 238 len 20480
204892024-02-27T21:51:59.196Z09/30 Read block 498 len 24576
204902024-02-27T21:51:59.196Z10/30 Write block 34 len 24576 data: 13 12 12 9 9 12
204912024-02-27T21:51:59.196Z11/30 Write block 72 len 36864 data: 15 12 13 14 16 15 13 12 12
204922024-02-27T21:51:59.196Z12/30 Write block 116 len 4096 data: 13
204932024-02-27T21:51:59.196Z13/30 Read block 170 len 36864
204942024-02-27T21:51:59.196Z14/30 Write block 130 len 16384 data: 14 12 8 10
204952024-02-27T21:51:59.196Z15/30 Write block 265 len 20480 data: 16 11 10 12 13
204962024-02-27T21:51:59.196Z16/30 Read block 404 len 28672
204972024-02-27T21:51:59.255Z17/30 Read block 175 len 8192
204982024-02-27T21:51:59.256Z18/30 Write block 278 len 8192 data: 12 11
204992024-02-27T21:51:59.256Z19/30 Write block 74 len 36864 data: 14 15 17 16 14 13 13 12 10
205002024-02-27T21:51:59.256Z20/30 Write block 177 len 32768 data: 14 17 16 17 20 21 18 18
205012024-02-27T21:51:59.256Z21/30 Flush
205022024-02-27T21:51:59.256Z22/30 Write block 262 len 12288 data: 14 14 17
205032024-02-27T21:51:59.257Z23/30 Write block 538 len 16384 data: 11 11 11 9
205042024-02-27T21:51:59.257Z24/30 Read block 448 len 4096
205052024-02-27T21:51:59.257Z25/30 Read block 88 len 16384
205062024-02-27T21:51:59.257Z26/30 Read block 468 len 36864
205072024-02-27T21:51:59.258Z27/30 Write block 459 len 16384 data: 10 11 12 13
205082024-02-27T21:51:59.258Z28/30 Write block 364 len 32768 data: 10 11 11 12 13 10 11 11
205092024-02-27T21:51:59.258Z29/30 Write block 29 len 8192 data: 11 14
205102024-02-27T21:51:59.258Z30/30 Read block 220 len 16384
205112024-02-27T21:51:59.258Z----------------------------------------------------------------
205122024-02-27T21:51:59.259Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
205132024-02-27T21:51:59.259ZStates: Active Active Active
205142024-02-27T21:51:59.260ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
205152024-02-27T21:51:59.261Z 1 Acked 1000 Write 6 Done Done Done false
205162024-02-27T21:51:59.261Z 2 Acked 1001 Write 5 Done Done Done false
205172024-02-27T21:51:59.261Z 3 Acked 1002 Write 7 Done Done Done false
205182024-02-27T21:51:59.261Z 4 Acked 1003 Read 10 Sent Done Done false
205192024-02-27T21:51:59.261Z 5 Acked 1004 Read 2 Sent Done Done false
205202024-02-27T21:51:59.261Z 6 Acked 1005 Flush 0 Sent Done Done false
205212024-02-27T21:51:59.262Z 7 Acked 1006 Write 8 Sent Done Done false
205222024-02-27T21:51:59.262Z 8 Acked 1007 Read 5 Sent Done Done false
205232024-02-27T21:51:59.262Z 9 Acked 1008 Read 6 Sent Done Done false
205242024-02-27T21:51:59.262Z 10 Acked 1009 Write 6 Sent Done Done false
205252024-02-27T21:51:59.263Z 11 Acked 1010 Write 9 Sent Done Done false
205262024-02-27T21:51:59.263Z 12 Acked 1011 Write 1 Sent Done Done false
205272024-02-27T21:51:59.263Z 13 Acked 1012 Read 9 Sent Done Done false
205282024-02-27T21:51:59.263Z 14 Acked 1013 Write 4 Sent Done Done false
205292024-02-27T21:51:59.263Z 15 Acked 1014 Write 5 Sent Done Done false
205302024-02-27T21:51:59.263Z 16 Acked 1015 Read 7 Sent Done Done false
205312024-02-27T21:51:59.264Z 17 Acked 1016 Read 2 Sent Done Done false
205322024-02-27T21:51:59.264Z 18 Acked 1017 Write 2 Sent Done Done false
205332024-02-27T21:51:59.264Z 19 Acked 1018 Write 9 Sent Done Done false
205342024-02-27T21:51:59.264Z 20 Acked 1019 Write 8 Sent Done Done false
205352024-02-27T21:51:59.264Z 21 Acked 1020 Flush 0 Sent Done Done false
205362024-02-27T21:51:59.265Z 22 Acked 1021 Write 3 Sent Done Done false
205372024-02-27T21:51:59.265Z 23 Acked 1022 Write 4 Sent Done Done false
205382024-02-27T21:51:59.265Z 24 Acked 1023 Read 1 Sent Done Done false
205392024-02-27T21:51:59.265Z 25 Acked 1024 Read 4 Sent Done Done false
205402024-02-27T21:51:59.265Z 26 Acked 1025 Read 9 Sent Done Done false
205412024-02-27T21:51:59.266Z 27 Acked 1026 Write 4 Sent Done Done false
205422024-02-27T21:51:59.266Z 28 Acked 1027 Write 8 Sent Done Done false
205432024-02-27T21:51:59.266Z 29 Acked 1028 Write 2 Sent Done Done false
205442024-02-27T21:51:59.266Z 30 Acked 1029 Read 4 Sent Done Done false
205452024-02-27T21:51:59.267Z STATES DS:0 DS:1 DS:2 TOTAL
205462024-02-27T21:51:59.267Z New 0 0 0 0
205472024-02-27T21:51:59.267Z Sent 27 0 0 27
205482024-02-27T21:51:59.267Z Done 3 30 30 63
205492024-02-27T21:51:59.267Z Skipped 0 0 0 0
205502024-02-27T21:51:59.268Z Error 0 0 0 0
205512024-02-27T21:51:59.268ZLast Flush: 0 1020 1020
205522024-02-27T21:51:59.269ZDownstairs last five completed:
205532024-02-27T21:51:59.269ZUpstairs last five completed: 30 29 28 27 26
205542024-02-27T21:51:59.269Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:51:57.473507683Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1910}
205552024-02-27T21:51:59.270ZWrote out file "/tmp/test_repair_verify.data"
205562024-02-27T21:51:59.270Z
205572024-02-27T21:51:59.270ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205582024-02-27T21:51:59.270ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205592024-02-27T21:51:59.270Z 0 000-019 188 188 188 337 337 337 F F F
205602024-02-27T21:51:59.271Z 1 020-039 184 190 190 331 339 339 F T T <---
205612024-02-27T21:51:59.271Z 2 040-059 179 179 179 323 323 323 F F F
205622024-02-27T21:51:59.271Z 3 060-079 186 190 190 334 339 339 F F F <---
205632024-02-27T21:51:59.272Z 4 080-099 186 190 190 334 339 339 F F F <---
205642024-02-27T21:51:59.272Z 5 100-119 188 190 190 337 339 339 F F F <---
205652024-02-27T21:51:59.272Z 6 120-139 188 190 190 337 339 339 F F F <---
205662024-02-27T21:51:59.272Z 7 140-159 180 180 180 325 325 325 F F F
205672024-02-27T21:51:59.272Z 8 160-179 182 190 190 328 339 339 T F F <---
205682024-02-27T21:51:59.273Z 9 180-199 182 190 190 328 339 339 T F F <---
205692024-02-27T21:51:59.273Z 10 200-219 185 185 185 332 332 332 F F F
205702024-02-27T21:51:59.273Z 11 220-239 186 190 190 335 338 338 T F F <---
205712024-02-27T21:51:59.273Z 12 240-259 186 190 190 335 338 338 T F F <---
205722024-02-27T21:51:59.274Z 13 260-279 185 190 190 332 339 339 F T T <---
205732024-02-27T21:51:59.274Z 14 280-299 185 185 185 332 332 332 F F F
205742024-02-27T21:51:59.274Z 15 300-319 188 188 188 337 337 337 F F F
205752024-02-27T21:51:59.274Z 16 320-339 186 186 186 335 335 335 F F F
205762024-02-27T21:51:59.274Z 17 340-359 175 190 190 315 339 339 F F F <---
205772024-02-27T21:51:59.275Z 18 360-379 188 188 188 337 337 337 F T T <---
205782024-02-27T21:51:59.275Z 19 380-399 188 188 188 337 337 337 F F F
205792024-02-27T21:51:59.275Z 20 400-419 185 185 185 332 332 332 F F F
205802024-02-27T21:51:59.275Z 21 420-439 187 187 187 336 336 336 F F F
205812024-02-27T21:51:59.276Z 22 440-459 188 188 188 337 337 337 F T T <---
205822024-02-27T21:51:59.276Z 23 460-479 187 187 187 336 336 336 F T T <---
205832024-02-27T21:51:59.276Z 24 480-499 188 188 188 337 337 337 F F F
205842024-02-27T21:51:59.276Z 25 500-519 188 188 188 337 337 337 F F F
205852024-02-27T21:51:59.276Z 26 520-539 188 188 188 337 337 337 F T T <---
205862024-02-27T21:51:59.277Z 27 540-559 184 184 184 330 330 330 F T T <---
205872024-02-27T21:51:59.277Z 28 560-579 188 188 188 337 337 337 F F F
205882024-02-27T21:51:59.277Z 29 580-599 188 188 188 337 337 337 F F F
205892024-02-27T21:51:59.277ZMax gen: 190, Max flush: 339
205902024-02-27T21:51:59.278ZError: Difference in extent metadata found!
205912024-02-27T21:51:59.278ZA Difference in extent metadata is expected here
205922024-02-27T21:51:59.278ZOn loop 95
205932024-02-27T21:51:59.278Z
205942024-02-27T21:51:59.278ZVerifying data now
205952024-02-27T21:52:08.796ZNo change after verify
205962024-02-27T21:52:08.818ZLoop: 95 Downstairs dump after verify (and repair):
205972024-02-27T21:52:08.819ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205982024-02-27T21:52:08.819Z 0 000-019 188 188 188 337 337 337 F F F
205992024-02-27T21:52:08.819Z 1 020-039 191 191 191 340 340 340 F F F
206002024-02-27T21:52:08.819Z 2 040-059 179 179 179 323 323 323 F F F
206012024-02-27T21:52:08.819Z 3 060-079 190 190 190 339 339 339 F F F
206022024-02-27T21:52:08.819Z 4 080-099 190 190 190 339 339 339 F F F
206032024-02-27T21:52:08.819Z 5 100-119 190 190 190 339 339 339 F F F
206042024-02-27T21:52:08.819Z 6 120-139 190 190 190 339 339 339 F F F
206052024-02-27T21:52:08.819Z 7 140-159 180 180 180 325 325 325 F F F
206062024-02-27T21:52:08.819Z 8 160-179 190 190 190 339 339 339 F F F
206072024-02-27T21:52:08.819Z 9 180-199 190 190 190 339 339 339 F F F
206082024-02-27T21:52:08.819Z 10 200-219 185 185 185 332 332 332 F F F
206092024-02-27T21:52:08.819Z 11 220-239 190 190 190 338 338 338 F F F
206102024-02-27T21:52:08.820Z 12 240-259 190 190 190 338 338 338 F F F
206112024-02-27T21:52:08.820Z 13 260-279 191 191 191 340 340 340 F F F
206122024-02-27T21:52:08.820Z 14 280-299 185 185 185 332 332 332 F F F
206132024-02-27T21:52:08.820Z 15 300-319 188 188 188 337 337 337 F F F
206142024-02-27T21:52:08.820Z 16 320-339 186 186 186 335 335 335 F F F
206152024-02-27T21:52:08.820Z 17 340-359 190 190 190 339 339 339 F F F
206162024-02-27T21:52:08.820Z 18 360-379 191 191 191 340 340 340 F F F
206172024-02-27T21:52:08.820Z 19 380-399 188 188 188 337 337 337 F F F
206182024-02-27T21:52:08.820Z 20 400-419 185 185 185 332 332 332 F F F
206192024-02-27T21:52:08.821Z 21 420-439 187 187 187 336 336 336 F F F
206202024-02-27T21:52:08.821Z 22 440-459 191 191 191 340 340 340 F F F
206212024-02-27T21:52:08.821Z 23 460-479 191 191 191 340 340 340 F F F
206222024-02-27T21:52:08.821Z 24 480-499 188 188 188 337 337 337 F F F
206232024-02-27T21:52:08.821Z 25 500-519 188 188 188 337 337 337 F F F
206242024-02-27T21:52:08.821Z 26 520-539 191 191 191 340 340 340 F F F
206252024-02-27T21:52:08.821Z 27 540-559 191 191 191 340 340 340 F F F
206262024-02-27T21:52:08.821Z 28 560-579 188 188 188 337 337 337 F F F
206272024-02-27T21:52:08.821Z 29 580-599 188 188 188 337 337 337 F F F
206282024-02-27T21:52:08.821ZMax gen: 191, Max flush: 340
206292024-02-27T21:52:08.821Z
206302024-02-27T21:52:08.821ZBegin loop 96
206312024-02-27T21:52:08.821ZDownstairs to restart: 0
206322024-02-27T21:52:08.822Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.342463835Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918}
206332024-02-27T21:52:08.822Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
206342024-02-27T21:52:08.822Z,"time":"2024-02-27T21:52:03.34284168Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918}
206352024-02-27T21:52:08.822Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.342878896Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918}
206362024-02-27T21:52:08.822Z{"msg":"Crucible a2ad824c-ea79-45cc-953b-d409c0cc75a8 has session id: 448bf1b0-63c4-46a9-95ff-6eaa905def9a","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.342920751Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206372024-02-27T21:52:08.822Z{"msg":"Upstairs opts: Upstairs UUID: a2ad824c-ea79-45cc-953b-d409c0cc75a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.342974265Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206382024-02-27T21:52:08.822Z{"msg":"Crucible stats registered with UUID: a2ad824c-ea79-45cc-953b-d409c0cc75a8","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.34301563Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206392024-02-27T21:52:08.822Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343043687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918}
206402024-02-27T21:52:08.822Z{"msg":"a2ad824c-ea79-45cc-953b-d409c0cc75a8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343060125Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206412024-02-27T21:52:08.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343087631Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206422024-02-27T21:52:08.822Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343141355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206432024-02-27T21:52:08.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343198728Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206442024-02-27T21:52:08.822Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343223446Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206452024-02-27T21:52:08.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343264921Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206462024-02-27T21:52:08.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343298627Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206472024-02-27T21:52:08.823Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343330893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206482024-02-27T21:52:08.823Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343398465Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206492024-02-27T21:52:08.823Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343446909Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206502024-02-27T21:52:08.823Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343492284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206512024-02-27T21:52:08.823Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.34352897Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206522024-02-27T21:52:08.823Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.343566355Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206532024-02-27T21:52:08.823Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.344079515Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206542024-02-27T21:52:08.823Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.344108291Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206552024-02-27T21:52:08.823Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.344141577Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206562024-02-27T21:52:08.823Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.344209019Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206572024-02-27T21:52:08.823Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.344694172Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206582024-02-27T21:52:08.824Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.344732038Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206592024-02-27T21:52:08.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.34497037Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206602024-02-27T21:52:08.824Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.345005166Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206612024-02-27T21:52:08.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.345259966Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206622024-02-27T21:52:08.824Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.34530851Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206632024-02-27T21:52:08.824Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.345337457Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206642024-02-27T21:52:08.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.345365803Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206652024-02-27T21:52:08.824Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.34538792Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206662024-02-27T21:52:08.824Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.345408258Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206672024-02-27T21:52:08.824Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.345554071Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206682024-02-27T21:52:08.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.345977951Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206692024-02-27T21:52:08.824Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346049253Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206702024-02-27T21:52:08.825Z{"msg":"[0]R flush_numbers[0..12]: [337, 340, 323, 339, 339, 339, 339, 325, 339, 339, 332, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346089178Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206712024-02-27T21:52:08.825Z{"msg":"[0]R generation[0..12]: [188, 191, 179, 190, 190, 190, 190, 180, 190, 190, 185, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346125034Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206722024-02-27T21:52:08.825Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346163309Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206732024-02-27T21:52:08.825Z{"msg":"Wait for a query_work_queue command to finish before sending IO
206742024-02-27T21:52:08.825Z[1]R flush_numbers[0..12]: [337, 340, 323, 339, 339, 339, 339, 325, 339, 339, 332, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346204854Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206752024-02-27T21:52:08.825Z{"msg":"[1]R generation[0..12]: [188, 191, 179, 190, 190, 190, 190, 180, 190, 190, 185, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.34624497Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206762024-02-27T21:52:08.825Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346284635Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206772024-02-27T21:52:08.825Z{"msg":"[2]R flush_numbers[0..12]: [337, 340, 323, 339, 339, 339, 339, 325, 339, 339, 332, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.34632515Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206782024-02-27T21:52:08.825Z{"msg":"[2]R generation[0..12]: [188, 191, 179, 190, 190, 190, 190, 180, 190, 190, 185, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346363765Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206792024-02-27T21:52:08.825Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346405501Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206802024-02-27T21:52:08.825Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346430068Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206812024-02-27T21:52:08.825Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346452285Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206822024-02-27T21:52:08.826Z{"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346466553Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206832024-02-27T21:52:08.826Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346486461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"":"downstairs","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206842024-02-27T21:52:08.826Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346506449Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206852024-02-27T21:52:08.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346520687Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"0","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206862024-02-27T21:52:08.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206872024-02-27T21:52:08.826Z,"time":"2024-02-27T21:52:03.346540545Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"1","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206882024-02-27T21:52:08.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346569811Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"client":"2","":"downstairs","session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206892024-02-27T21:52:08.826Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346591339Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206902024-02-27T21:52:08.826Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346611227Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206912024-02-27T21:52:08.826Z{"msg":"a2ad824c-ea79-45cc-953b-d409c0cc75a8 is now active with session: 448bf1b0-63c4-46a9-95ff-6eaa905def9a","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346631044Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206922024-02-27T21:52:08.826Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.346649982Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918,"session_id":"448bf1b0-63c4-46a9-95ff-6eaa905def9a"}
206932024-02-27T21:52:08.826Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:03.34666871Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918}
206942024-02-27T21:52:08.826ZLoading write count information from file "/tmp/test_repair_verify.data"
206952024-02-27T21:52:08.827ZRun Repair workload
206962024-02-27T21:52:08.839ZError: Difference in extent metadata found!
206972024-02-27T21:52:08.839Z01/30 Write block 271 len 16384 data: 15 17 13 13
206982024-02-27T21:52:08.839Z02/30 Write block 128 len 20480 data: 16 15 15 13 9
206992024-02-27T21:52:08.839Z03/30 Write block 1 len 4096 data: 5
207002024-02-27T21:52:08.839Z04/30 Write block 524 len 24576 data: 15 13 15 17 15 14
207012024-02-27T21:52:08.839Z05/30 Read block 36 len 32768
207022024-02-27T21:52:08.839Z06/30 Read block 334 len 24576
207032024-02-27T21:52:08.839Z07/30 Read block 477 len 40960
207042024-02-27T21:52:08.840Z08/30 Read block 248 len 20480
207052024-02-27T21:52:08.840Z09/30 Flush
207062024-02-27T21:52:08.840Z10/30 Flush
207072024-02-27T21:52:08.840Z11/30 Write block 78 len 32768 data: 15 14 14 13 11 9 11 12
207082024-02-27T21:52:08.840Z12/30 Write block 521 len 4096 data: 15
207092024-02-27T21:52:08.840Z13/30 Read block 213 len 20480
207102024-02-27T21:52:08.840Z14/30 Write block 194 len 12288 data: 14 12 11
207112024-02-27T21:52:08.840Z15/30 Read block 434 len 36864
207122024-02-27T21:52:08.840Z16/30 Write block 295 len 16384 data: 21 18 18 15
207132024-02-27T21:52:08.840Z17/30 Write block 383 len 40960 data: 15 14 15 12 14 12 14 14 16 14
207142024-02-27T21:52:08.840Z18/30 Read block 348 len 36864
207152024-02-27T21:52:08.840Z19/30 Write block 191 len 4096 data: 16
207162024-02-27T21:52:08.840Z20/30 Read block 348 len 40960
207172024-02-27T21:52:08.840Z21/30 Read block 111 len 32768
207182024-02-27T21:52:08.840Z22/30 Flush
207192024-02-27T21:52:08.840Z23/30 Write block 162 len 32768 data: 15 16 14 12 9 12 11 13
207202024-02-27T21:52:08.841Z24/30 Read block 446 len 12288
207212024-02-27T21:52:08.841Z25/30 Read block 59 len 32768
207222024-02-27T21:52:08.841Z26/30 Read block 174 len 20480
207232024-02-27T21:52:08.841Z27/30 Read block 503 len 24576
207242024-02-27T21:52:08.841Z28/30 Write block 566 len 16384 data: 11 12 13 14
207252024-02-27T21:52:08.841Z29/30 Write block 133 len 16384 data: 11 10 12 11
207262024-02-27T21:52:08.841Z30/30 Write block 65 len 24576 data: 10 10 11 11 10 12
207272024-02-27T21:52:08.841Z----------------------------------------------------------------
207282024-02-27T21:52:08.841Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
207292024-02-27T21:52:08.841ZStates: Active Active Active
207302024-02-27T21:52:08.841ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
207312024-02-27T21:52:08.841Z 1 Acked 1000 Write 4 Sent Done Done false
207322024-02-27T21:52:08.841Z 2 Acked 1001 Write 5 Sent Done Done false
207332024-02-27T21:52:08.841Z 3 Acked 1002 Write 1 Sent Done Done false
207342024-02-27T21:52:08.841Z 4 Acked 1003 Write 6 Sent Done Done false
207352024-02-27T21:52:08.841Z 5 Acked 1004 Read 8 Sent Done Done false
207362024-02-27T21:52:08.841Z 6 Acked 1005 Read 6 Sent Done Done false
207372024-02-27T21:52:08.841Z 7 Acked 1006 Read 10 Sent Done Done false
207382024-02-27T21:52:08.841Z 8 Acked 1007 Read 5 Sent Done Done false
207392024-02-27T21:52:08.841Z 9 Acked 1008 Flush 0 Sent Done Done false
207402024-02-27T21:52:08.841Z 10 Acked 1009 Flush 0 Sent Done Done false
207412024-02-27T21:52:08.841Z 11 Acked 1010 Write 8 Sent Done Done false
207422024-02-27T21:52:08.841Z 12 Acked 1011 Write 1 Sent Done Done false
207432024-02-27T21:52:08.841Z 13 Acked 1012 Read 5 Sent Done Done false
207442024-02-27T21:52:08.842Z 14 Acked 1013 Write 3 Sent Done Done false
207452024-02-27T21:52:08.842Z 15 Acked 1014 Read 9 Sent Done Done false
207462024-02-27T21:52:08.842Z 16 Acked 1015 Write 4 Sent Done Done false
207472024-02-27T21:52:08.842Z 17 Acked 1016 Write 10 Sent Done Done false
207482024-02-27T21:52:08.842Z 18 Acked 1017 Read 9 Sent Done Done false
207492024-02-27T21:52:08.842Z 19 Acked 1018 Write 1 Sent Done Done false
207502024-02-27T21:52:08.842Z 20 Acked 1019 Read 10 Sent Done Done false
207512024-02-27T21:52:08.842Z 21 Acked 1020 Read 8 Sent Done Done false
207522024-02-27T21:52:08.842Z 22 Acked 1021 Flush 0 Sent Done Done false
207532024-02-27T21:52:08.842Z 23 Acked 1022 Write 8 Sent Done Done false
207542024-02-27T21:52:08.842Z 24 Acked 1023 Read 3 Sent Done Done false
207552024-02-27T21:52:08.842Z 25 Acked 1024 Read 8 Sent Done Done false
207562024-02-27T21:52:08.842Z 26 Acked 1025 Read 5 Sent Done Done false
207572024-02-27T21:52:08.842Z 27 Acked 1026 Read 6 Sent Done Done false
207582024-02-27T21:52:08.842Z 28 Acked 1027 Write 4 Sent Done Done false
207592024-02-27T21:52:08.842Z 29 Acked 1028 Write 4 Sent Done Done false
207602024-02-27T21:52:08.842Z 30 Acked 1029 Write 6 Sent Sent Sent false
207612024-02-27T21:52:08.842Z STATES DS:0 DS:1 DS:2 TOTAL
207622024-02-27T21:52:08.842Z New 0 0 0 0
207632024-02-27T21:52:08.842Z Sent 30 1 1 32
207642024-02-27T21:52:08.842Z Done 0 29 29 58
207652024-02-27T21:52:08.842Z Skipped 0 0 0 0
207662024-02-27T21:52:08.842Z Error 0 0 0 0
207672024-02-27T21:52:08.842ZLast Flush: 0 1021 1021
207682024-02-27T21:52:08.843ZDownstairs last five completed:
207692024-02-27T21:52:08.843ZUpstairs last five completed: 30 29 28 27 26
207702024-02-27T21:52:08.843Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:52:03.377331448Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1918}
207712024-02-27T21:52:08.843ZWrote out file "/tmp/test_repair_verify.data"
207722024-02-27T21:52:08.843Z
207732024-02-27T21:52:08.843ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207742024-02-27T21:52:08.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207752024-02-27T21:52:08.843Z 0 000-019 188 192 192 337 341 341 F F F <---
207762024-02-27T21:52:08.843Z 1 020-039 191 191 191 340 340 340 F F F
207772024-02-27T21:52:08.843Z 2 040-059 179 179 179 323 323 323 F F F
207782024-02-27T21:52:08.843Z 3 060-079 190 192 192 339 343 343 F T T <---
207792024-02-27T21:52:08.843Z 4 080-099 190 192 192 339 343 343 F F F <---
207802024-02-27T21:52:08.843Z 5 100-119 190 190 190 339 339 339 F F F
207812024-02-27T21:52:08.843Z 6 120-139 190 192 192 339 341 341 F T T <---
207822024-02-27T21:52:08.843Z 7 140-159 180 180 180 325 325 325 F F F
207832024-02-27T21:52:08.843Z 8 160-179 190 190 190 339 339 339 F T T <---
207842024-02-27T21:52:08.843Z 9 180-199 190 192 192 339 343 343 F F F <---
207852024-02-27T21:52:08.843Z 10 200-219 185 185 185 332 332 332 F F F
207862024-02-27T21:52:08.844Z 11 220-239 190 190 190 338 338 338 F F F
207872024-02-27T21:52:08.844Z 12 240-259 190 190 190 338 338 338 F F F
207882024-02-27T21:52:08.844Z 13 260-279 191 192 192 340 341 341 F F F <---
207892024-02-27T21:52:08.844Z 14 280-299 185 192 192 332 343 343 F F F <---
207902024-02-27T21:52:08.844Z 15 300-319 188 188 188 337 337 337 F F F
207912024-02-27T21:52:08.844Z 16 320-339 186 186 186 335 335 335 F F F
207922024-02-27T21:52:08.844Z 17 340-359 190 190 190 339 339 339 F F F
207932024-02-27T21:52:08.844Z 18 360-379 191 191 191 340 340 340 F F F
207942024-02-27T21:52:08.844Z 19 380-399 188 192 192 337 343 343 F F F <---
207952024-02-27T21:52:08.844Z 20 400-419 185 185 185 332 332 332 F F F
207962024-02-27T21:52:08.871Z 21 420-439 187 187 187 336 336 336 F F F
207972024-02-27T21:52:08.871Z 22 440-459 191 191 191 340 340 340 F F F
207982024-02-27T21:52:08.872Z 23 460-479 191 191 191 340 340 340 F F F
207992024-02-27T21:52:08.872Z 24 480-499 188 188 188 337 337 337 F F F
208002024-02-27T21:52:08.872Z 25 500-519 188 188 188 337 337 337 F F F
208012024-02-27T21:52:08.872Z 26 520-539 191 192 192 340 343 343 F F F <---
208022024-02-27T21:52:08.872Z 27 540-559 191 191 191 340 340 340 F F F
208032024-02-27T21:52:08.873Z 28 560-579 188 188 188 337 337 337 F T T <---
208042024-02-27T21:52:08.873Z 29 580-599 188 188 188 337 337 337 F F F
208052024-02-27T21:52:08.873ZMax gen: 192, Max flush: 343
208062024-02-27T21:52:08.873ZA Difference in extent metadata is expected here
208072024-02-27T21:52:08.873ZOn loop 96
208082024-02-27T21:52:08.873Z
208092024-02-27T21:52:08.873ZVerifying data now
208102024-02-27T21:52:09.624ZNo change after verify
208112024-02-27T21:52:09.624ZLoop: 96 Downstairs dump after verify (and repair):
208122024-02-27T21:52:09.639ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208132024-02-27T21:52:09.639Z 0 000-019 192 192 192 341 341 341 F F F
208142024-02-27T21:52:09.639Z 1 020-039 191 191 191 340 340 340 F F F
208152024-02-27T21:52:09.639Z 2 040-059 179 179 179 323 323 323 F F F
208162024-02-27T21:52:09.639Z 3 060-079 193 193 193 344 344 344 F F F
208172024-02-27T21:52:09.639Z 4 080-099 192 192 192 343 343 343 F F F
208182024-02-27T21:52:09.639Z 5 100-119 190 190 190 339 339 339 F F F
208192024-02-27T21:52:09.639Z 6 120-139 193 193 193 344 344 344 F F F
208202024-02-27T21:52:09.639Z 7 140-159 180 180 180 325 325 325 F F F
208212024-02-27T21:52:09.652Z 8 160-179 193 193 193 344 344 344 F F F
208222024-02-27T21:52:09.652Z 9 180-199 192 192 192 343 343 343 F F F
208232024-02-27T21:52:09.652Z 10 200-219 185 185 185 332 332 332 F F F
208242024-02-27T21:52:09.652Z 11 220-239 190 190 190 338 338 338 F F F
208252024-02-27T21:52:09.652Z 12 240-259 190 190 190 338 338 338 F F F
208262024-02-27T21:52:09.652Z 13 260-279 192 192 192 341 341 341 F F F
208272024-02-27T21:52:09.652Z 14 280-299 192 192 192 343 343 343 F F F
208282024-02-27T21:52:09.652Z 15 300-319 188 188 188 337 337 337 F F F
208292024-02-27T21:52:09.652Z 16 320-339 186 186 186 335 335 335 F F F
208302024-02-27T21:52:09.652Z 17 340-359 190 190 190 339 339 339 F F F
208312024-02-27T21:52:09.652Z 18 360-379 191 191 191 340 340 340 F F F
208322024-02-27T21:52:09.652Z 19 380-399 192 192 192 343 343 343 F F F
208332024-02-27T21:52:09.652Z 20 400-419 185 185 185 332 332 332 F F F
208342024-02-27T21:52:09.652Z 21 420-439 187 187 187 336 336 336 F F F
208352024-02-27T21:52:09.652Z 22 440-459 191 191 191 340 340 340 F F F
208362024-02-27T21:52:09.652Z 23 460-479 191 191 191 340 340 340 F F F
208372024-02-27T21:52:09.652Z 24 480-499 188 188 188 337 337 337 F F F
208382024-02-27T21:52:09.653Z 25 500-519 188 188 188 337 337 337 F F F
208392024-02-27T21:52:09.653Z 26 520-539 192 192 192 343 343 343 F F F
208402024-02-27T21:52:09.653Z 27 540-559 191 191 191 340 340 340 F F F
208412024-02-27T21:52:09.653Z 28 560-579 193 193 193 344 344 344 F F F
208422024-02-27T21:52:09.653Z 29 580-599 188 188 188 337 337 337 F F F
208432024-02-27T21:52:09.653ZMax gen: 193, Max flush: 344
208442024-02-27T21:52:09.653Z
208452024-02-27T21:52:09.653ZBegin loop 97
208462024-02-27T21:52:09.653ZDownstairs to restart: 0
208472024-02-27T21:52:09.668Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.014514893Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926}
208482024-02-27T21:52:09.668Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\nCrucible runtime is spawned
208492024-02-27T21:52:09.668Z git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.01487671Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926}
208502024-02-27T21:52:09.669Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.014924025Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926}
208512024-02-27T21:52:09.685Z{"msg":"Crucible ac6b1ffb-ed75-45d5-b639-2d5e2c480384 has session id: b70dc888-e971-49aa-9086-a2470599c6bf","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.014953691Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208522024-02-27T21:52:09.685Z{"msg":"Upstairs opts: Upstairs UUID: ac6b1ffb-ed75-45d5-b639-2d5e2c480384, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015018973Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208532024-02-27T21:52:09.685Z{"msg":"Crucible stats registered with UUID: ac6b1ffb-ed75-45d5-b639-2d5e2c480384","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015057159Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208542024-02-27T21:52:09.685Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015091285Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926}
208552024-02-27T21:52:09.685Z{"msg":"ac6b1ffb-ed75-45d5-b639-2d5e2c480384 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015157937Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208562024-02-27T21:52:09.686Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015212311Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208572024-02-27T21:52:09.686Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015267804Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208582024-02-27T21:52:09.686Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015319548Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208592024-02-27T21:52:09.686Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015367192Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208602024-02-27T21:52:09.686Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015438254Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208612024-02-27T21:52:09.686Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015491358Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208622024-02-27T21:52:09.686Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015521364Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208632024-02-27T21:52:09.686Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015567429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208642024-02-27T21:52:09.686Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015598945Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208652024-02-27T21:52:09.686Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.01564058Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208662024-02-27T21:52:09.686Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015716081Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208672024-02-27T21:52:09.686Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.015748798Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208682024-02-27T21:52:09.687Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.016427807Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208692024-02-27T21:52:09.687Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.016478521Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208702024-02-27T21:52:09.687Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.016522476Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208712024-02-27T21:52:09.687Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.0165773Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208722024-02-27T21:52:09.687Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.016714274Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208732024-02-27T21:52:09.687Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.016784676Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208742024-02-27T21:52:09.687Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.017162121Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208752024-02-27T21:52:09.687Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.017200116Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208762024-02-27T21:52:09.687Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.017465495Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208772024-02-27T21:52:09.687Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.017739403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208782024-02-27T21:52:09.687Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.017793027Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208792024-02-27T21:52:09.687Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.017823413Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208802024-02-27T21:52:09.687Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.017846251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208812024-02-27T21:52:09.687Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.017860949Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208822024-02-27T21:52:09.688Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.017883176Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208832024-02-27T21:52:09.688Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018029989Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208842024-02-27T21:52:09.688Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018073374Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208852024-02-27T21:52:09.688Z{"msg":"[0]R flush_numbers[0..12]: [341, 340, 323, 344, 343, 339, 344, 325, 344, 343, 332, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018091451Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208862024-02-27T21:52:09.688Z{"msg":"[0]R generation[0..12]: [192, 191, 179, 193, 192, 190, 193, 180, 193, 192, 185, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018115709Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208872024-02-27T21:52:09.688Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018131097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208882024-02-27T21:52:09.688Z{"msg":"[1]R flush_numbers[0..12]: [341, 340, 323, 344, 343, 339, 344, 325, 344, 343, 332, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018152304Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208892024-02-27T21:52:09.688Z{"msg":"[1]R generation[0..12]: [192, 191, 179, 193, 192, 190, 193, 180, 193, 192, 185, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018167053Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208902024-02-27T21:52:09.688Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
208912024-02-27T21:52:09.688Zcrucible","level":30,"time":"2024-02-27T21:52:08.01819094Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208922024-02-27T21:52:09.688Z{"msg":"[2]R flush_numbers[0..12]: [341, 340, 323, 344, 343, 339, 344, 325, 344, 343, 332, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018205988Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208932024-02-27T21:52:09.688Z{"msg":"[2]R generation[0..12]: [192, 191, 179, 193, 192, 190, 193, 180, 193, 192, 185, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018227375Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208942024-02-27T21:52:09.688Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018241984Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208952024-02-27T21:52:09.689Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018261962Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208962024-02-27T21:52:09.689Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.01827564Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208972024-02-27T21:52:09.689Z{"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018295918Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208982024-02-27T21:52:09.689Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018309186Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"":"downstairs","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
208992024-02-27T21:52:09.689Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018329863Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
209002024-02-27T21:52:09.689Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018342472Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"0","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
209012024-02-27T21:52:09.689Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.01836216Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"1","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
209022024-02-27T21:52:09.689Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018375788Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"client":"2","":"downstairs","session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
209032024-02-27T21:52:09.689Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018396096Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
209042024-02-27T21:52:09.689Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018409104Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
209052024-02-27T21:52:09.689Z{"msg":"ac6b1ffb-ed75-45d5-b639-2d5e2c480384 is now active with session: b70dc888-e971-49aa-9086-a2470599c6bf","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018428742Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
209062024-02-27T21:52:09.689Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.01844105Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926,"session_id":"b70dc888-e971-49aa-9086-a2470599c6bf"}
209072024-02-27T21:52:09.690Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:08.018459748Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926}
209082024-02-27T21:52:09.690ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
209092024-02-27T21:52:09.690ZLoading write count information from file "/tmp/test_repair_verify.data"
209102024-02-27T21:52:09.690ZRun Repair workload
209112024-02-27T21:52:09.690Z01/30 Write block 33 len 4096 data: 12
209122024-02-27T21:52:09.690Z02/30 Read block 393 len 20480
209132024-02-27T21:52:09.690Z03/30 Write block 350 len 20480 data: 20 19 16 14 13
209142024-02-27T21:52:09.690Z04/30 Read block 247 len 36864
209152024-02-27T21:52:09.690Z05/30 Write block 69 len 20480 data: 11 13 12 16 13
209162024-02-27T21:52:09.690Z06/30 Write block 591 len 12288 data: 12 15 16
209172024-02-27T21:52:09.690Z07/30 Write block 363 len 24576 data: 12 11 12 12 13 14
209182024-02-27T21:52:09.691Z08/30 Write block 454 len 32768 data: 10 7 5 3 5 11 12 13
209192024-02-27T21:52:09.691Z09/30 Read block 182 len 8192
209202024-02-27T21:52:09.691Z10/30 Write block 261 len 40960 data: 13 15 15 18 17 12 11 13 14 14
209212024-02-27T21:52:09.691Z11/30 Write block 65 len 4096 data: 11
209222024-02-27T21:52:09.691Z12/30 Write block 501 len 24576 data: 9 11 11 12 14 15
209232024-02-27T21:52:09.691Z13/30 Flush
209242024-02-27T21:52:09.691Z14/30 Write block 261 len 8192 data: 14 16
209252024-02-27T21:52:09.697Z15/30 Read block 490 len 12288
209262024-02-27T21:52:09.697Z16/30 Write block 528 len 4096 data: 16
209272024-02-27T21:52:09.697Z17/30 Read block 191 len 12288
209282024-02-27T21:52:09.697Z18/30 Write block 200 len 20480 data: 14 12 11 11 9
209292024-02-27T21:52:09.697Z19/30 Write block 305 len 20480 data: 13 15 17 16 17
209302024-02-27T21:52:09.697Z20/30 Flush
209312024-02-27T21:52:09.697Z21/30 Write block 287 len 36864 data: 11 12 16 16 18 17 21 20 22
209322024-02-27T21:52:09.697Z22/30 Read block 359 len 8192
209332024-02-27T21:52:09.698Z23/30 Read block 537 len 16384
209342024-02-27T21:52:09.698Z24/30 Read block 178 len 32768
209352024-02-27T21:52:09.698Z25/30 Read block 578 len 24576
209362024-02-27T21:52:09.698Z26/30 Flush
209372024-02-27T21:52:09.698Z27/30 Write block 187 len 24576 data: 15 19 16 14 17 15
209382024-02-27T21:52:09.704Z28/30 Write block 342 len 4096 data: 17
209392024-02-27T21:52:09.704Z29/30 Read block 280 len 28672
209402024-02-27T21:52:09.705Z30/30 Write block 167 len 24576 data: 13 12 14 15 15 15
209412024-02-27T21:52:09.705Z----------------------------------------------------------------
209422024-02-27T21:52:09.705Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
209432024-02-27T21:52:09.705ZStates: Active Active Active
209442024-02-27T21:52:09.705ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
209452024-02-27T21:52:09.706Z 1 Acked 1000 Write 1 Sent Done Done false
209462024-02-27T21:52:09.706Z 2 Acked 1001 Read 5 Sent Done Done false
209472024-02-27T21:52:09.706Z 3 Acked 1002 Write 5 Sent Done Done false
209482024-02-27T21:52:09.706Z 4 Acked 1003 Read 9 Sent Done Done false
209492024-02-27T21:52:09.707Z 5 Acked 1004 Write 5 Sent Done Done false
209502024-02-27T21:52:09.707Z 6 Acked 1005 Write 3 Sent Done Done false
209512024-02-27T21:52:09.707Z 7 Acked 1006 Write 6 Sent Done Done false
209522024-02-27T21:52:09.707Z 8 Acked 1007 Write 8 Sent Done Done false
209532024-02-27T21:52:09.708Z 9 Acked 1008 Read 2 Sent Done Done false
209542024-02-27T21:52:09.708Z 10 Acked 1009 Write 10 Sent Done Done false
209552024-02-27T21:52:09.708Z 11 Acked 1010 Write 1 Sent Done Done false
209562024-02-27T21:52:09.708Z 12 Acked 1011 Write 6 Sent Done Done false
209572024-02-27T21:52:09.709Z 13 Acked 1012 Flush 0 Sent Done Done false
209582024-02-27T21:52:09.709Z 14 Acked 1013 Write 2 Sent Done Done false
209592024-02-27T21:52:09.709Z 15 Acked 1014 Read 3 Sent Done Done false
209602024-02-27T21:52:09.709Z 16 Acked 1015 Write 1 Sent Done Done false
209612024-02-27T21:52:09.709Z 17 Acked 1016 Read 3 Sent Done Done false
209622024-02-27T21:52:09.710Z 18 Acked 1017 Write 5 Sent Done Done false
209632024-02-27T21:52:09.710Z 19 Acked 1018 Write 5 Sent Done Done false
209642024-02-27T21:52:09.710Z 20 Acked 1019 Flush 0 Sent Done Done false
209652024-02-27T21:52:09.711Z 21 Acked 1020 Write 9 Sent Done Done false
209662024-02-27T21:52:09.711Z 22 Acked 1021 Read 2 Sent Done Done false
209672024-02-27T21:52:09.711Z 23 Acked 1022 Read 4 Sent Done Done false
209682024-02-27T21:52:09.711Z 24 Acked 1023 Read 8 Sent Done Done false
209692024-02-27T21:52:09.711Z 25 Acked 1024 Read 6 Sent Done Done false
209702024-02-27T21:52:09.711Z 26 Acked 1025 Flush 0 Sent Done Done false
209712024-02-27T21:52:09.711Z 27 Acked 1026 Write 6 Sent Done Done false
209722024-02-27T21:52:09.712Z 28 Acked 1027 Write 1 Sent Done Done false
209732024-02-27T21:52:09.712Z 29 Acked 1028 Read 7 Sent Done Done false
209742024-02-27T21:52:09.712Z 30 Acked 1029 Write 6 Sent Sent Sent false
209752024-02-27T21:52:09.712Z STATES DS:0 DS:1 DS:2 TOTAL
209762024-02-27T21:52:09.712Z New 0 0 0 0
209772024-02-27T21:52:09.712Z Sent 30 1 1 32
209782024-02-27T21:52:09.712Z Done 0 29 29 58
209792024-02-27T21:52:09.712Z Skipped 0 0 0 0
209802024-02-27T21:52:09.712Z Error 0 0 0 0
209812024-02-27T21:52:09.713ZLast Flush: 0 1025 1025
209822024-02-27T21:52:09.713ZDownstairs last five completed:
209832024-02-27T21:52:09.713ZUpstairs last five completed: 30 29 28 27 26
209842024-02-27T21:52:09.713Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:52:08.050244184Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1926}
209852024-02-27T21:52:09.713ZWrote out file "/tmp/test_repair_verify.data"
209862024-02-27T21:52:09.733Z
209872024-02-27T21:52:09.733ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209882024-02-27T21:52:09.744ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209892024-02-27T21:52:09.744Z 0 000-019 192 192 192 341 341 341 F F F
209902024-02-27T21:52:09.744Z 1 020-039 191 194 194 340 345 345 F F F <---
209912024-02-27T21:52:09.744Z 2 040-059 179 179 179 323 323 323 F F F
209922024-02-27T21:52:09.744Z 3 060-079 193 194 194 344 345 345 F F F <---
209932024-02-27T21:52:09.744Z 4 080-099 192 192 192 343 343 343 F F F
209942024-02-27T21:52:09.744Z 5 100-119 190 190 190 339 339 339 F F F
209952024-02-27T21:52:09.745Z 6 120-139 193 193 193 344 344 344 F F F
209962024-02-27T21:52:09.745Z 7 140-159 180 180 180 325 325 325 F F F
209972024-02-27T21:52:09.745Z 8 160-179 193 193 193 344 344 344 F T T <---
209982024-02-27T21:52:09.745Z 9 180-199 192 192 192 343 343 343 F T T <---
209992024-02-27T21:52:09.745Z 10 200-219 185 194 194 332 346 346 F F F <---
210002024-02-27T21:52:09.745Z 11 220-239 190 190 190 338 338 338 F F F
210012024-02-27T21:52:09.745Z 12 240-259 190 190 190 338 338 338 F F F
210022024-02-27T21:52:09.745Z 13 260-279 192 194 194 341 346 346 F F F <---
210032024-02-27T21:52:09.745Z 14 280-299 192 194 194 343 347 347 F F F <---
210042024-02-27T21:52:09.745Z 15 300-319 188 194 194 337 346 346 F F F <---
210052024-02-27T21:52:09.745Z 16 320-339 186 186 186 335 335 335 F F F
210062024-02-27T21:52:09.745Z 17 340-359 190 194 194 339 345 345 F T T <---
210072024-02-27T21:52:09.745Z 18 360-379 191 194 194 340 345 345 F F F <---
210082024-02-27T21:52:09.745Z 19 380-399 192 192 192 343 343 343 F F F
210092024-02-27T21:52:09.745Z 20 400-419 185 185 185 332 332 332 F F F
210102024-02-27T21:52:09.745Z 21 420-439 187 187 187 336 336 336 F F F
210112024-02-27T21:52:09.745Z 22 440-459 191 194 194 340 345 345 F F F <---
210122024-02-27T21:52:09.745Z 23 460-479 191 194 194 340 345 345 F F F <---
210132024-02-27T21:52:09.746Z 24 480-499 188 188 188 337 337 337 F F F
210142024-02-27T21:52:09.746Z 25 500-519 188 194 194 337 345 345 F F F <---
210152024-02-27T21:52:09.746Z 26 520-539 192 194 194 343 346 346 F F F <---
210162024-02-27T21:52:09.746Z 27 540-559 191 191 191 340 340 340 F F F
210172024-02-27T21:52:09.746Z 28 560-579 193 193 193 344 344 344 F F F
210182024-02-27T21:52:09.746Z 29 580-599 188 194 194 337 345 345 F F F <---
210192024-02-27T21:52:09.746ZMax gen: 194, Max flush: 347
210202024-02-27T21:52:09.746ZError: Difference in extent metadata found!
210212024-02-27T21:52:09.746ZA Difference in extent metadata is expected here
210222024-02-27T21:52:09.746ZOn loop 97
210232024-02-27T21:52:09.746Z
210242024-02-27T21:52:09.746ZVerifying data now
210252024-02-27T21:52:14.471ZNo change after verify
210262024-02-27T21:52:14.471ZLoop: 97 Downstairs dump after verify (and repair):
210272024-02-27T21:52:14.484ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210282024-02-27T21:52:14.484Z 0 000-019 192 192 192 341 341 341 F F F
210292024-02-27T21:52:14.484Z 1 020-039 194 194 194 345 345 345 F F F
210302024-02-27T21:52:14.484Z 2 040-059 179 179 179 323 323 323 F F F
210312024-02-27T21:52:14.484Z 3 060-079 194 194 194 345 345 345 F F F
210322024-02-27T21:52:14.484Z 4 080-099 192 192 192 343 343 343 F F F
210332024-02-27T21:52:14.484Z 5 100-119 190 190 190 339 339 339 F F F
210342024-02-27T21:52:14.484Z 6 120-139 193 193 193 344 344 344 F F F
210352024-02-27T21:52:14.484Z 7 140-159 180 180 180 325 325 325 F F F
210362024-02-27T21:52:14.489Z 8 160-179 195 195 195 348 348 348 F F F
210372024-02-27T21:52:14.489Z 9 180-199 195 195 195 348 348 348 F F F
210382024-02-27T21:52:14.489Z 10 200-219 194 194 194 346 346 346 F F F
210392024-02-27T21:52:14.489Z 11 220-239 190 190 190 338 338 338 F F F
210402024-02-27T21:52:14.489Z 12 240-259 190 190 190 338 338 338 F F F
210412024-02-27T21:52:14.489Z 13 260-279 194 194 194 346 346 346 F F F
210422024-02-27T21:52:14.489Z 14 280-299 194 194 194 347 347 347 F F F
210432024-02-27T21:52:14.489Z 15 300-319 194 194 194 346 346 346 F F F
210442024-02-27T21:52:14.489Z 16 320-339 186 186 186 335 335 335 F F F
210452024-02-27T21:52:14.489Z 17 340-359 195 195 195 348 348 348 F F F
210462024-02-27T21:52:14.489Z 18 360-379 194 194 194 345 345 345 F F F
210472024-02-27T21:52:14.489Z 19 380-399 192 192 192 343 343 343 F F F
210482024-02-27T21:52:14.489Z 20 400-419 185 185 185 332 332 332 F F F
210492024-02-27T21:52:14.489Z 21 420-439 187 187 187 336 336 336 F F F
210502024-02-27T21:52:14.489Z 22 440-459 194 194 194 345 345 345 F F F
210512024-02-27T21:52:14.489Z 23 460-479 194 194 194 345 345 345 F F F
210522024-02-27T21:52:14.490Z 24 480-499 188 188 188 337 337 337 F F F
210532024-02-27T21:52:14.490Z 25 500-519 194 194 194 345 345 345 F F F
210542024-02-27T21:52:14.490Z 26 520-539 194 194 194 346 346 346 F F F
210552024-02-27T21:52:14.490Z 27 540-559 191 191 191 340 340 340 F F F
210562024-02-27T21:52:14.490Z 28 560-579 193 193 193 344 344 344 F F F
210572024-02-27T21:52:14.490Z 29 580-599 194 194 194 345 345 345 F F F
210582024-02-27T21:52:14.490ZMax gen: 195, Max flush: 348
210592024-02-27T21:52:14.490Z
210602024-02-27T21:52:14.490ZBegin loop 98
210612024-02-27T21:52:14.490ZDownstairs to restart: 0
210622024-02-27T21:52:14.507Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.856773191Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934}
210632024-02-27T21:52:14.507Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857019093Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934}
210642024-02-27T21:52:14.507Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857047829Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934}
210652024-02-27T21:52:14.518Z{"msg":"Crucible f9d87786-75db-4c12-ac70-1b26e78fd281 has session id: 68fbe226-11ba-434a-b077-7b7fae745129","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857069117Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210662024-02-27T21:52:14.518Z{"msg":"Upstairs opts: Upstairs UUID: f9d87786-75db-4c12-ac70-1b26e78fd281, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
210672024-02-27T21:52:14.518Z,"time":"2024-02-27T21:52:12.857113661Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210682024-02-27T21:52:14.518Z{"msg":"Crucible stats registered with UUID: f9d87786-75db-4c12-ac70-1b26e78fd281","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857157826Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210692024-02-27T21:52:14.518Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857192972Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934}
210702024-02-27T21:52:14.518Z{"msg":"f9d87786-75db-4c12-ac70-1b26e78fd281 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857235967Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210712024-02-27T21:52:14.518Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857260024Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210722024-02-27T21:52:14.518Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857288141Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210732024-02-27T21:52:14.518Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857303429Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210742024-02-27T21:52:14.518Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857324446Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210752024-02-27T21:52:14.518Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857338555Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210762024-02-27T21:52:14.518Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857360612Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210772024-02-27T21:52:14.518Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857393338Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210782024-02-27T21:52:14.519Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857419066Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210792024-02-27T21:52:14.519Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857442683Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210802024-02-27T21:52:14.519Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857569128Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210812024-02-27T21:52:14.519Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.857608173Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210822024-02-27T21:52:14.519Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.85763525Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210832024-02-27T21:52:14.519Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.858295282Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210842024-02-27T21:52:14.519Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.858336127Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210852024-02-27T21:52:14.519Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.858351696Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210862024-02-27T21:52:14.519Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.858366114Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210872024-02-27T21:52:14.519Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.858387971Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210882024-02-27T21:52:14.519Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.85840196Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210892024-02-27T21:52:14.519Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.85899042Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210902024-02-27T21:52:14.519Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859069091Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210912024-02-27T21:52:14.519Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859130994Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210922024-02-27T21:52:14.519Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.85958801Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210932024-02-27T21:52:14.519Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859634614Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210942024-02-27T21:52:14.519Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859665211Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210952024-02-27T21:52:14.519Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859704426Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210962024-02-27T21:52:14.519Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859722514Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210972024-02-27T21:52:14.519Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859744461Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210982024-02-27T21:52:14.519Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.85975849Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
210992024-02-27T21:52:14.519Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859779617Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211002024-02-27T21:52:14.519Z{"msg":"[0]R flush_numbers[0..12]: [341, 345, 323, 345, 343, 339, 344, 325, 348, 348, 346, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859794536Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211012024-02-27T21:52:14.520Z{"msg":"[0]R generation[0..12]: [192, 194, 179, 194, 192, 190, 193, 180, 195, 195, 194, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859815883Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211022024-02-27T21:52:14.520Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859831281Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211032024-02-27T21:52:14.520Z{"msg":"[1]R flush_numbers[0..12]: [341, 345, 323, 345, 343, 339, 344, 325, 348, 348, 346, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859853619Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211042024-02-27T21:52:14.520Z{"msg":"[1]R generation[0..12]: [192, 194, 179, 194, 192, 190, 193, 180, 195, 195, 194, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859868257Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211052024-02-27T21:52:14.520ZWait for a query_work_queue command to finish before sending IO
211062024-02-27T21:52:14.520Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.85993033Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211072024-02-27T21:52:14.520Z{"msg":"[2]R flush_numbers[0..12]: [341, 345, 323, 345, 343, 339, 344, 325, 348, 348, 346, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859959516Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211082024-02-27T21:52:14.520Z{"msg":"[2]R generation[0..12]: [192, 194, 179, 194, 192, 190, 193, 180, 195, 195, 194, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.859977234Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211092024-02-27T21:52:14.520Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.86001224Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211102024-02-27T21:52:14.520Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860032618Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211112024-02-27T21:52:14.520Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860055895Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211122024-02-27T21:52:14.520Z{"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860075062Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211132024-02-27T21:52:14.520Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860104399Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"":"downstairs","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211142024-02-27T21:52:14.520Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860134156Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211152024-02-27T21:52:14.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860157583Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"0","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211162024-02-27T21:52:14.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860177161Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"1","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211172024-02-27T21:52:14.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860200718Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"client":"2","":"downstairs","session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211182024-02-27T21:52:14.520Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860232284Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211192024-02-27T21:52:14.520Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860257911Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211202024-02-27T21:52:14.520Z{"msg":"f9d87786-75db-4c12-ac70-1b26e78fd281 is now active with session: 68fbe226-11ba-434a-b077-7b7fae745129","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.860285948Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211212024-02-27T21:52:14.520Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
211222024-02-27T21:52:14.520Z,"time":"2024-02-27T21:52:12.860317474Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934,"session_id":"68fbe226-11ba-434a-b077-7b7fae745129"}
211232024-02-27T21:52:14.520Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:12.86035333Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934}
211242024-02-27T21:52:14.520ZLoading write count information from file "/tmp/test_repair_verify.data"
211252024-02-27T21:52:14.520ZRun Repair workload
211262024-02-27T21:52:14.520Z01/30 Write block 17 len 8192 data: 18 19
211272024-02-27T21:52:14.526Z02/30 Write block 184 len 4096 data: 19
211282024-02-27T21:52:14.526Z03/30 Read block 594 len 20480
211292024-02-27T21:52:14.526Z04/30 Flush
211302024-02-27T21:52:14.526Z05/30 Write block 290 len 12288 data: 17 19 18
211312024-02-27T21:52:14.526Z06/30 Read block 169 len 20480
211322024-02-27T21:52:14.526Z07/30 Write block 299 len 24576 data: 12 14 15 17 15 17
211332024-02-27T21:52:14.526Z08/30 Flush
211342024-02-27T21:52:14.526Z09/30 Write block 412 len 36864 data: 11 12 12 15 16 20 18 17 12
211352024-02-27T21:52:14.526Z10/30 Write block 187 len 8192 data: 16 20
211362024-02-27T21:52:14.526Z11/30 Read block 128 len 12288
211372024-02-27T21:52:14.526Z12/30 Write block 223 len 32768 data: 15 15 12 13 15 15 14 17
211382024-02-27T21:52:14.526Z13/30 Read block 348 len 8192
211392024-02-27T21:52:14.526Z14/30 Read block 423 len 8192
211402024-02-27T21:52:14.530Z15/30 Write block 347 len 20480 data: 17 22 21 21 20
211412024-02-27T21:52:14.530Z16/30 Write block 129 len 24576 data: 16 16 14 10 12 11
211422024-02-27T21:52:14.530Z17/30 Write block 564 len 24576 data: 15 12 12 13 14 15
211432024-02-27T21:52:14.530Z18/30 Write block 201 len 12288 data: 13 12 12
211442024-02-27T21:52:14.530Z19/30 Read block 472 len 4096
211452024-02-27T21:52:14.531Z20/30 Read block 572 len 4096
211462024-02-27T21:52:14.531Z21/30 Read block 24 len 16384
211472024-02-27T21:52:14.531Z22/30 Read block 263 len 8192
211482024-02-27T21:52:14.534Z23/30 Read block 283 len 24576
211492024-02-27T21:52:14.534Z24/30 Read block 392 len 32768
211502024-02-27T21:52:14.535Z25/30 Read block 384 len 36864
211512024-02-27T21:52:14.535Z26/30 Write block 393 len 32768 data: 15 13 14 13 16 14 13 17
211522024-02-27T21:52:14.542Z27/30 Write block 141 len 20480 data: 15 17 15 12 11
211532024-02-27T21:52:14.542Z28/30 Read block 509 len 16384
211542024-02-27T21:52:14.542Z29/30 Write block 560 len 4096 data: 17
211552024-02-27T21:52:14.542Z30/30 Read block 570 len 8192
211562024-02-27T21:52:14.542Z----------------------------------------------------------------
211572024-02-27T21:52:14.542Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
211582024-02-27T21:52:14.542ZStates: Active Active Active
211592024-02-27T21:52:14.542ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211602024-02-27T21:52:14.542Z 1 Acked 1000 Write 2 Done Done Done false
211612024-02-27T21:52:14.542Z 2 Acked 1001 Write 1 Done Done Done false
211622024-02-27T21:52:14.542Z 3 Acked 1002 Read 5 Sent Done Done false
211632024-02-27T21:52:14.542Z 4 Acked 1003 Flush 0 Sent Done Done false
211642024-02-27T21:52:14.542Z 5 Acked 1004 Write 3 Sent Done Done false
211652024-02-27T21:52:14.542Z 6 Acked 1005 Read 5 Sent Done Done false
211662024-02-27T21:52:14.542Z 7 Acked 1006 Write 6 Sent Done Done false
211672024-02-27T21:52:14.542Z 8 Acked 1007 Flush 0 Sent Done Done false
211682024-02-27T21:52:14.542Z 9 Acked 1008 Write 9 Sent Done Done false
211692024-02-27T21:52:14.542Z 10 Acked 1009 Write 2 Sent Done Done false
211702024-02-27T21:52:14.543Z 11 Acked 1010 Read 3 Sent Done Done false
211712024-02-27T21:52:14.543Z 12 Acked 1011 Write 8 Sent Done Done false
211722024-02-27T21:52:14.543Z 13 Acked 1012 Read 2 Sent Done Done false
211732024-02-27T21:52:14.543Z 14 Acked 1013 Read 2 Sent Done Done false
211742024-02-27T21:52:14.543Z 15 Acked 1014 Write 5 Sent Done Done false
211752024-02-27T21:52:14.543Z 16 Acked 1015 Write 6 Sent Done Done false
211762024-02-27T21:52:14.543Z 17 Acked 1016 Write 6 Sent Done Done false
211772024-02-27T21:52:14.543Z 18 Acked 1017 Write 3 Sent Done Done false
211782024-02-27T21:52:14.543Z 19 Acked 1018 Read 1 Sent Done Done false
211792024-02-27T21:52:14.543Z 20 Acked 1019 Read 1 Sent Done Done false
211802024-02-27T21:52:14.543Z 21 Acked 1020 Read 4 Sent Done Done false
211812024-02-27T21:52:14.543Z 22 Acked 1021 Read 2 Sent Done Done false
211822024-02-27T21:52:14.543Z 23 Acked 1022 Read 6 Sent Done Done false
211832024-02-27T21:52:14.543Z 24 Acked 1023 Read 8 Sent Done Done false
211842024-02-27T21:52:14.543Z 25 Acked 1024 Read 9 Sent Done Done false
211852024-02-27T21:52:14.543Z 26 Acked 1025 Write 8 Sent Done Done false
211862024-02-27T21:52:14.543Z 27 Acked 1026 Write 5 Sent Done Done false
211872024-02-27T21:52:14.543Z 28 Acked 1027 Read 4 Sent Done Done false
211882024-02-27T21:52:14.543Z 29 Acked 1028 Write 1 Sent Done Done false
211892024-02-27T21:52:14.543Z 30 Acked 1029 Read 2 Sent Done Done false
211902024-02-27T21:52:14.543Z STATES DS:0 DS:1 DS:2 TOTAL
211912024-02-27T21:52:14.543Z New 0 0 0 0
211922024-02-27T21:52:14.543Z Sent 28 0 0 28
211932024-02-27T21:52:14.543Z Done 2 30 30 62
211942024-02-27T21:52:14.543Z Skipped 0 0 0 0
211952024-02-27T21:52:14.544Z Error 0 0 0 0
211962024-02-27T21:52:14.544ZLast Flush: 0 1007 1007
211972024-02-27T21:52:14.544ZDownstairs last five completed:
211982024-02-27T21:52:14.544ZUpstairs last five completed: 30 29 28 27 26
211992024-02-27T21:52:14.544Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:52:12.887574793Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1934}
212002024-02-27T21:52:14.544ZWrote out file "/tmp/test_repair_verify.data"
212012024-02-27T21:52:14.548Z
212022024-02-27T21:52:14.548ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
212032024-02-27T21:52:14.566ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212042024-02-27T21:52:14.566Z 0 000-019 192 196 196 341 349 349 T F F <---
212052024-02-27T21:52:14.566Z 1 020-039 194 194 194 345 345 345 F F F
212062024-02-27T21:52:14.566Z 2 040-059 179 179 179 323 323 323 F F F
212072024-02-27T21:52:14.566Z 3 060-079 194 194 194 345 345 345 F F F
212082024-02-27T21:52:14.566Z 4 080-099 192 192 192 343 343 343 F F F
212092024-02-27T21:52:14.567Z 5 100-119 190 190 190 339 339 339 F F F
212102024-02-27T21:52:14.567Z 6 120-139 193 193 193 344 344 344 F T T <---
212112024-02-27T21:52:14.574Z 7 140-159 180 180 180 325 325 325 F T T <---
212122024-02-27T21:52:14.574Z 8 160-179 195 195 195 348 348 348 F F F
212132024-02-27T21:52:14.574Z 9 180-199 195 196 196 348 349 349 T T T <---
212142024-02-27T21:52:14.574Z 10 200-219 194 194 194 346 346 346 F T T <---
212152024-02-27T21:52:14.574Z 11 220-239 190 190 190 338 338 338 F T T <---
212162024-02-27T21:52:14.574Z 12 240-259 190 190 190 338 338 338 F F F
212172024-02-27T21:52:14.574Z 13 260-279 194 194 194 346 346 346 F F F
212182024-02-27T21:52:14.575Z 14 280-299 194 196 196 347 350 350 F F F <---
212192024-02-27T21:52:14.575Z 15 300-319 194 196 196 346 350 350 F F F <---
212202024-02-27T21:52:14.575Z 16 320-339 186 186 186 335 335 335 F F F
212212024-02-27T21:52:14.575Z 17 340-359 195 195 195 348 348 348 F T T <---
212222024-02-27T21:52:14.575Z 18 360-379 194 194 194 345 345 345 F F F
212232024-02-27T21:52:14.575Z 19 380-399 192 192 192 343 343 343 F T T <---
212242024-02-27T21:52:14.575Z 20 400-419 185 185 185 332 332 332 F T T <---
212252024-02-27T21:52:14.575Z 21 420-439 187 187 187 336 336 336 F T T <---
212262024-02-27T21:52:14.575Z 22 440-459 194 194 194 345 345 345 F F F
212272024-02-27T21:52:14.575Z 23 460-479 194 194 194 345 345 345 F F F
212282024-02-27T21:52:14.575Z 24 480-499 188 188 188 337 337 337 F F F
212292024-02-27T21:52:14.575Z 25 500-519 194 194 194 345 345 345 F F F
212302024-02-27T21:52:14.575Z 26 520-539 194 194 194 346 346 346 F F F
212312024-02-27T21:52:14.575Z 27 540-559 191 191 191 340 340 340 F F F
212322024-02-27T21:52:14.575Z 28 560-579 193 193 193 344 344 344 F T T <---
212332024-02-27T21:52:14.575Z 29 580-599 194 194 194 345 345 345 F F F
212342024-02-27T21:52:14.575ZMax gen: 196, Max flush: 350
212352024-02-27T21:52:14.575ZError: Difference in extent metadata found!
212362024-02-27T21:52:14.575ZA Difference in extent metadata is expected here
212372024-02-27T21:52:14.575ZOn loop 98
212382024-02-27T21:52:14.575Z
212392024-02-27T21:52:14.575ZVerifying data now
212402024-02-27T21:52:19.248ZNo change after verify
212412024-02-27T21:52:19.248ZLoop: 98 Downstairs dump after verify (and repair):
212422024-02-27T21:52:19.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212432024-02-27T21:52:19.267Z 0 000-019 196 196 196 349 349 349 F F F
212442024-02-27T21:52:19.267Z 1 020-039 194 194 194 345 345 345 F F F
212452024-02-27T21:52:19.267Z 2 040-059 179 179 179 323 323 323 F F F
212462024-02-27T21:52:19.267Z 3 060-079 194 194 194 345 345 345 F F F
212472024-02-27T21:52:19.267Z 4 080-099 192 192 192 343 343 343 F F F
212482024-02-27T21:52:19.268Z 5 100-119 190 190 190 339 339 339 F F F
212492024-02-27T21:52:19.268Z 6 120-139 197 197 197 351 351 351 F F F
212502024-02-27T21:52:19.268Z 7 140-159 197 197 197 351 351 351 F F F
212512024-02-27T21:52:19.268Z 8 160-179 195 195 195 348 348 348 F F F
212522024-02-27T21:52:19.268Z 9 180-199 197 197 197 351 351 351 F F F
212532024-02-27T21:52:19.268Z 10 200-219 197 197 197 351 351 351 F F F
212542024-02-27T21:52:19.268Z 11 220-239 197 197 197 351 351 351 F F F
212552024-02-27T21:52:19.268Z 12 240-259 190 190 190 338 338 338 F F F
212562024-02-27T21:52:19.268Z 13 260-279 194 194 194 346 346 346 F F F
212572024-02-27T21:52:19.268Z 14 280-299 196 196 196 350 350 350 F F F
212582024-02-27T21:52:19.268Z 15 300-319 196 196 196 350 350 350 F F F
212592024-02-27T21:52:19.268Z 16 320-339 186 186 186 335 335 335 F F F
212602024-02-27T21:52:19.268Z 17 340-359 197 197 197 351 351 351 F F F
212612024-02-27T21:52:19.268Z 18 360-379 194 194 194 345 345 345 F F F
212622024-02-27T21:52:19.268Z 19 380-399 197 197 197 351 351 351 F F F
212632024-02-27T21:52:19.268Z 20 400-419 197 197 197 351 351 351 F F F
212642024-02-27T21:52:19.268Z 21 420-439 197 197 197 351 351 351 F F F
212652024-02-27T21:52:19.268Z 22 440-459 194 194 194 345 345 345 F F F
212662024-02-27T21:52:19.268Z 23 460-479 194 194 194 345 345 345 F F F
212672024-02-27T21:52:19.269Z 24 480-499 188 188 188 337 337 337 F F F
212682024-02-27T21:52:19.269Z 25 500-519 194 194 194 345 345 345 F F F
212692024-02-27T21:52:19.269Z 26 520-539 194 194 194 346 346 346 F F F
212702024-02-27T21:52:19.269Z 27 540-559 191 191 191 340 340 340 F F F
212712024-02-27T21:52:19.269Z 28 560-579 197 197 197 351 351 351 F F F
212722024-02-27T21:52:19.269Z 29 580-599 194 194 194 345 345 345 F F F
212732024-02-27T21:52:19.269ZMax gen: 197, Max flush: 351
212742024-02-27T21:52:19.274Z
212752024-02-27T21:52:19.274ZBegin loop 99
212762024-02-27T21:52:19.274ZDownstairs to restart: 2
212772024-02-27T21:52:19.274Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.621641787Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942}
212782024-02-27T21:52:19.274Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f6c40444ff68361c835e88ee91c409e8deb48d7d\",\n git_commit_timestamp: \"2024-02-27T21:23:06.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.621971018Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942}
212792024-02-27T21:52:19.274Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
212802024-02-27T21:52:19.274Z,"time":"2024-02-27T21:52:17.621998525Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942}
212812024-02-27T21:52:19.274Z{"msg":"Crucible 45cab1df-91d3-40e2-be7c-ddff2604a0cd has session id: efc56998-c2f0-4ef3-bbb0-5bd7091e594d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.62204099Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212822024-02-27T21:52:19.274Z{"msg":"Upstairs opts: Upstairs UUID: 45cab1df-91d3-40e2-be7c-ddff2604a0cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622088914Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212832024-02-27T21:52:19.274Z{"msg":"Crucible stats registered with UUID: 45cab1df-91d3-40e2-be7c-ddff2604a0cd","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622148127Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212842024-02-27T21:52:19.274Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622194042Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942}
212852024-02-27T21:52:19.274Z{"msg":"45cab1df-91d3-40e2-be7c-ddff2604a0cd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.62221037Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212862024-02-27T21:52:19.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622246746Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212872024-02-27T21:52:19.274Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622265873Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212882024-02-27T21:52:19.274Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622303069Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212892024-02-27T21:52:19.275Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622324697Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212902024-02-27T21:52:19.275Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622338885Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212912024-02-27T21:52:19.275Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622386489Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212922024-02-27T21:52:19.275Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622439323Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212932024-02-27T21:52:19.275Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622482508Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212942024-02-27T21:52:19.275Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622511504Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212952024-02-27T21:52:19.275Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622537112Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212962024-02-27T21:52:19.275Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.622561129Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212972024-02-27T21:52:19.275Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.62263085Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212982024-02-27T21:52:19.280Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.62322514Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
212992024-02-27T21:52:19.280Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.623264996Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213002024-02-27T21:52:19.280Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.623291983Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213012024-02-27T21:52:19.280Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.62331411Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213022024-02-27T21:52:19.280Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.623338917Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213032024-02-27T21:52:19.280Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.623369063Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213042024-02-27T21:52:19.280Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.623811551Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213052024-02-27T21:52:19.280Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.623850427Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213062024-02-27T21:52:19.280Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.623969033Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213072024-02-27T21:52:19.280Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624436678Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213082024-02-27T21:52:19.280Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624488082Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213092024-02-27T21:52:19.280Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624527097Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213102024-02-27T21:52:19.280Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624565183Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213112024-02-27T21:52:19.280Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.62458919Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213122024-02-27T21:52:19.280Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624620196Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213132024-02-27T21:52:19.280Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624642813Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213142024-02-27T21:52:19.280Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.62467049Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213152024-02-27T21:52:19.280Z{"msg":"[0]R flush_numbers[0..12]: [349, 345, 323, 345, 343, 339, 351, 351, 348, 351, 351, 351]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624695767Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213162024-02-27T21:52:19.281Z{"msg":"[0]R generation[0..12]: [196, 194, 179, 194, 192, 190, 197, 197, 195, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624723624Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213172024-02-27T21:52:19.281Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624749421Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213182024-02-27T21:52:19.281ZWait for a query_work_queue command to finish before sending IO
213192024-02-27T21:52:19.281Z{"msg":"[1]R flush_numbers[0..12]: [349, 345, 323, 345, 343, 339, 351, 351, 348, 351, 351, 351]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624780737Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213202024-02-27T21:52:19.281Z{"msg":"[1]R generation[0..12]: [196, 194, 179, 194, 192, 190, 197, 197, 195, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624806804Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213212024-02-27T21:52:19.281Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624836821Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213222024-02-27T21:52:19.281Z{"msg":"[2]R flush_numbers[0..12]: [349, 345, 323, 345, 343, 339, 351, 351, 348, 351, 351, 351]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624879476Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213232024-02-27T21:52:19.281Z{"msg":"[2]R generation[0..12]: [196, 194, 179, 194, 192, 190, 197, 197, 195, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624913072Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213242024-02-27T21:52:19.281Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624934229Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213252024-02-27T21:52:19.281Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624963896Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213262024-02-27T21:52:19.281Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.624981524Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213272024-02-27T21:52:19.281Z{"msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625023459Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213282024-02-27T21:52:19.281Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625039417Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"":"downstairs","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213292024-02-27T21:52:19.281Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625061634Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213302024-02-27T21:52:19.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625074403Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"0","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213312024-02-27T21:52:19.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625088251Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"1","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213322024-02-27T21:52:19.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625109458Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"client":"2","":"downstairs","session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213332024-02-27T21:52:19.281Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625123377Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213342024-02-27T21:52:19.281Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213352024-02-27T21:52:19.281Z,"time":"2024-02-27T21:52:17.625136475Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213362024-02-27T21:52:19.281Z{"msg":"45cab1df-91d3-40e2-be7c-ddff2604a0cd is now active with session: efc56998-c2f0-4ef3-bbb0-5bd7091e594d","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625164362Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213372024-02-27T21:52:19.281Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625176591Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942,"session_id":"efc56998-c2f0-4ef3-bbb0-5bd7091e594d"}
213382024-02-27T21:52:19.281Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-27T21:52:17.625196408Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942}
213392024-02-27T21:52:19.281ZLoading write count information from file "/tmp/test_repair_verify.data"
213402024-02-27T21:52:19.281ZRun Repair workload
213412024-02-27T21:52:19.282Z01/30 Write block 307 len 20480 data: 18 17 18 19 20
213422024-02-27T21:52:19.282Z02/30 Read block 514 len 8192
213432024-02-27T21:52:19.282Z03/30 Write block 342 len 4096 data: 18
213442024-02-27T21:52:19.285Z04/30 Read block 513 len 32768
213452024-02-27T21:52:19.285Z05/30 Read block 256 len 16384
213462024-02-27T21:52:19.285Z06/30 Write block 278 len 40960 data: 13 12 10 9 9 8 8 7 8 12
213472024-02-27T21:52:19.285Z07/30 Read block 496 len 20480
213482024-02-27T21:52:19.285Z08/30 Read block 310 len 8192
213492024-02-27T21:52:19.285Z09/30 Write block 245 len 40960 data: 13 15 15 14 15 15 16 16 15 14
213502024-02-27T21:52:19.285Z10/30 Read block 563 len 24576
213512024-02-27T21:52:19.289Z11/30 Flush
213522024-02-27T21:52:19.289Z12/30 Write block 285 len 4096 data: 8
213532024-02-27T21:52:19.289Z13/30 Read block 201 len 28672
213542024-02-27T21:52:19.296Z14/30 Read block 426 len 24576
213552024-02-27T21:52:19.296Z15/30 Write block 357 len 8192 data: 11 12
213562024-02-27T21:52:19.296Z16/30 Flush
213572024-02-27T21:52:19.296Z17/30 Write block 351 len 4096 data: 21
213582024-02-27T21:52:19.296Z18/30 Write block 289 len 32768 data: 17 18 20 19 22 21 23 19
213592024-02-27T21:52:19.296Z19/30 Flush
213602024-02-27T21:52:19.300Z20/30 Write block 212 len 40960 data: 16 15 17 17 17 19 19 17 15 15
213612024-02-27T21:52:19.300Z21/30 Flush
213622024-02-27T21:52:19.303Z22/30 Write block 11 len 16384 data: 18 18 16 16
213632024-02-27T21:52:19.303Z23/30 Flush
213642024-02-27T21:52:19.307Z24/30 Write block 129 len 36864 data: 17 17 15 11 13 12 13 12 13
213652024-02-27T21:52:19.307Z25/30 Flush
213662024-02-27T21:52:19.307Z26/30 Write block 237 len 4096 data: 13
213672024-02-27T21:52:19.307Z27/30 Write block 101 len 8192 data: 12 15
213682024-02-27T21:52:19.311Z28/30 Read block 81 len 8192
213692024-02-27T21:52:19.311Z29/30 Write block 525 len 36864 data: 14 16 18 17 15 12 12 10 12
213702024-02-27T21:52:19.312Z30/30 Read block 564 len 12288
213712024-02-27T21:52:19.312Z----------------------------------------------------------------
213722024-02-27T21:52:19.312Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
213732024-02-27T21:52:19.312ZStates: Active Active Active
213742024-02-27T21:52:19.312ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213752024-02-27T21:52:19.312Z 1 Acked 1000 Write 5 Done Done Sent false
213762024-02-27T21:52:19.312Z 2 Acked 1001 Read 2 Done Done Sent false
213772024-02-27T21:52:19.312Z 3 Acked 1002 Write 1 Done Done Sent false
213782024-02-27T21:52:19.312Z 4 Acked 1003 Read 8 Done Done Sent false
213792024-02-27T21:52:19.312Z 5 Acked 1004 Read 4 Done Done Sent false
213802024-02-27T21:52:19.312Z 6 Acked 1005 Write 10 Done Done Sent false
213812024-02-27T21:52:19.312Z 7 Acked 1006 Read 5 Done Done Sent false
213822024-02-27T21:52:19.312Z 8 Acked 1007 Read 2 Done Done Sent false
213832024-02-27T21:52:19.313Z 9 Acked 1008 Write 10 Done Done Sent false
213842024-02-27T21:52:19.313Z 10 Acked 1009 Read 6 Done Done Sent false
213852024-02-27T21:52:19.313Z 11 Acked 1010 Flush 0 Done Done Sent false
213862024-02-27T21:52:19.313Z 12 Acked 1011 Write 1 Done Done Sent false
213872024-02-27T21:52:19.313Z 13 Acked 1012 Read 7 Done Done Sent false
213882024-02-27T21:52:19.313Z 14 Acked 1013 Read 6 Done Done Sent false
213892024-02-27T21:52:19.313Z 15 Acked 1014 Write 2 Done Done Sent false
213902024-02-27T21:52:19.313Z 16 Acked 1015 Flush 0 Done Done Sent false
213912024-02-27T21:52:19.313Z 17 Acked 1016 Write 1 Done Done Sent false
213922024-02-27T21:52:19.313Z 18 Acked 1017 Write 8 Done Done Sent false
213932024-02-27T21:52:19.313Z 19 Acked 1018 Flush 0 Done Done Sent false
213942024-02-27T21:52:19.313Z 20 Acked 1019 Write 10 Done Done Sent false
213952024-02-27T21:52:19.313Z 21 Acked 1020 Flush 0 Done Done Sent false
213962024-02-27T21:52:19.314Z 22 Acked 1021 Write 4 Done Done Sent false
213972024-02-27T21:52:19.314Z 23 Acked 1022 Flush 0 Done Done Sent false
213982024-02-27T21:52:19.314Z 24 Acked 1023 Write 9 Done Done Sent false
213992024-02-27T21:52:19.314Z 25 Acked 1024 Flush 0 Done Done Sent false
214002024-02-27T21:52:19.314Z 26 Acked 1025 Write 1 Done Done Sent false
214012024-02-27T21:52:19.314Z 27 Acked 1026 Write 2 Done Done Sent false
214022024-02-27T21:52:19.314Z 28 Acked 1027 Read 2 Done Done Sent false
214032024-02-27T21:52:19.314Z 29 Acked 1028 Write 9 Done Done Sent false
214042024-02-27T21:52:19.314Z 30 Acked 1029 Read 3 Done Done Sent false
214052024-02-27T21:52:19.314Z STATES DS:0 DS:1 DS:2 TOTAL
214062024-02-27T21:52:19.314Z New 0 0 0 0
214072024-02-27T21:52:19.314Z Sent 0 0 30 30
214082024-02-27T21:52:19.314Z Done 30 30 0 60
214092024-02-27T21:52:19.315Z Skipped 0 0 0 0
214102024-02-27T21:52:19.315Z Error 0 0 0 0
214112024-02-27T21:52:19.315ZLast Flush: 1024 1024 0
214122024-02-27T21:52:19.315ZDownstairs last five completed:
214132024-02-27T21:52:19.315ZUpstairs last five completed: 30 29 28 27 26
214142024-02-27T21:52:19.315Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-27T21:52:17.660310252Z","hostname":"ip-10-150-1-196.us-west-2.compute.internal","pid":1942}
214152024-02-27T21:52:19.315ZWrote out file "/tmp/test_repair_verify.data"
214162024-02-27T21:52:19.319Z
214172024-02-27T21:52:19.319ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
214182024-02-27T21:52:19.335ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214192024-02-27T21:52:19.336Z 0 000-019 198 198 196 356 356 349 F F F <---
214202024-02-27T21:52:19.336Z 1 020-039 194 194 194 345 345 345 F F F
214212024-02-27T21:52:19.336Z 2 040-059 179 179 179 323 323 323 F F F
214222024-02-27T21:52:19.336Z 3 060-079 194 194 194 345 345 345 F F F
214232024-02-27T21:52:19.336Z 4 080-099 192 192 192 343 343 343 F F F
214242024-02-27T21:52:19.336Z 5 100-119 190 190 190 339 339 339 T T F <---
214252024-02-27T21:52:19.336Z 6 120-139 198 198 197 357 357 351 F F F <---
214262024-02-27T21:52:19.336Z 7 140-159 197 197 197 351 351 351 F F F
214272024-02-27T21:52:19.336Z 8 160-179 195 195 195 348 348 348 F F F
214282024-02-27T21:52:19.336Z 9 180-199 197 197 197 351 351 351 F F F
214292024-02-27T21:52:19.336Z 10 200-219 198 198 197 355 355 351 F F F <---
214302024-02-27T21:52:19.336Z 11 220-239 198 198 197 355 355 351 T T F <---
214312024-02-27T21:52:19.337Z 12 240-259 198 198 190 352 352 338 F F F <---
214322024-02-27T21:52:19.340Z 13 260-279 198 198 194 352 352 346 F F F <---
214332024-02-27T21:52:19.341Z 14 280-299 198 198 196 354 354 350 F F F <---
214342024-02-27T21:52:19.341Z 15 300-319 198 198 196 352 352 350 F F F <---
214352024-02-27T21:52:19.341Z 16 320-339 186 186 186 335 335 335 F F F
214362024-02-27T21:52:19.341Z 17 340-359 198 198 197 354 354 351 F F F <---
214372024-02-27T21:52:19.341Z 18 360-379 194 194 194 345 345 345 F F F
214382024-02-27T21:52:19.341Z 19 380-399 197 197 197 351 351 351 F F F
214392024-02-27T21:52:19.341Z 20 400-419 197 197 197 351 351 351 F F F
214402024-02-27T21:52:19.341Z 21 420-439 197 197 197 351 351 351 F F F
214412024-02-27T21:52:19.341Z 22 440-459 194 194 194 345 345 345 F F F
214422024-02-27T21:52:19.341Z 23 460-479 194 194 194 345 345 345 F F F
214432024-02-27T21:52:19.341Z 24 480-499 188 188 188 337 337 337 F F F
214442024-02-27T21:52:19.341Z 25 500-519 194 194 194 345 345 345 F F F
214452024-02-27T21:52:19.341Z 26 520-539 194 194 194 346 346 346 T T F <---
214462024-02-27T21:52:19.342Z 27 540-559 191 191 191 340 340 340 F F F
214472024-02-27T21:52:19.342Z 28 560-579 197 197 197 351 351 351 F F F
214482024-02-27T21:52:19.342Z 29 580-599 194 194 194 345 345 345 F F F
214492024-02-27T21:52:19.342ZMax gen: 198, Max flush: 357
214502024-02-27T21:52:19.342ZError: Difference in extent metadata found!
214512024-02-27T21:52:19.342ZA Difference in extent metadata is expected here
214522024-02-27T21:52:19.342ZOn loop 99
214532024-02-27T21:52:19.342Z
214542024-02-27T21:52:19.342ZVerifying data now
214552024-02-27T21:52:23.911ZNo change after verify
214562024-02-27T21:52:23.911ZLoop: 99 Downstairs dump after verify (and repair):
214572024-02-27T21:52:23.925ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214582024-02-27T21:52:23.926Z 0 000-019 198 198 198 356 356 356 F F F
214592024-02-27T21:52:23.926Z 1 020-039 194 194 194 345 345 345 F F F
214602024-02-27T21:52:23.926Z 2 040-059 179 179 179 323 323 323 F F F
214612024-02-27T21:52:23.926Z 3 060-079 194 194 194 345 345 345 F F F
214622024-02-27T21:52:23.926Z 4 080-099 192 192 192 343 343 343 F F F
214632024-02-27T21:52:23.926Z 5 100-119 199 199 199 358 358 358 F F F
214642024-02-27T21:52:23.926Z 6 120-139 198 198 198 357 357 357 F F F
214652024-02-27T21:52:23.926Z 7 140-159 197 197 197 351 351 351 F F F
214662024-02-27T21:52:23.926Z 8 160-179 195 195 195 348 348 348 F F F
214672024-02-27T21:52:23.926Z 9 180-199 197 197 197 351 351 351 F F F
214682024-02-27T21:52:23.932Z 10 200-219 198 198 198 355 355 355 F F F
214692024-02-27T21:52:23.932Z 11 220-239 199 199 199 358 358 358 F F F
214702024-02-27T21:52:23.932Z 12 240-259 198 198 198 352 352 352 F F F
214712024-02-27T21:52:23.932Z 13 260-279 198 198 198 352 352 352 F F F
214722024-02-27T21:52:23.932Z 14 280-299 198 198 198 354 354 354 F F F
214732024-02-27T21:52:23.932Z 15 300-319 198 198 198 352 352 352 F F F
214742024-02-27T21:52:23.932Z 16 320-339 186 186 186 335 335 335 F F F
214752024-02-27T21:52:23.932Z 17 340-359 198 198 198 354 354 354 F F F
214762024-02-27T21:52:23.933Z 18 360-379 194 194 194 345 345 345 F F F
214772024-02-27T21:52:23.933Z 19 380-399 197 197 197 351 351 351 F F F
214782024-02-27T21:52:23.933Z 20 400-419 197 197 197 351 351 351 F F F
214792024-02-27T21:52:23.933Z 21 420-439 197 197 197 351 351 351 F F F
214802024-02-27T21:52:23.933Z 22 440-459 194 194 194 345 345 345 F F F
214812024-02-27T21:52:23.933Z 23 460-479 194 194 194 345 345 345 F F F
214822024-02-27T21:52:23.933Z 24 480-499 188 188 188 337 337 337 F F F
214832024-02-27T21:52:23.933Z 25 500-519 194 194 194 345 345 345 F F F
214842024-02-27T21:52:23.933Z 26 520-539 199 199 199 358 358 358 F F F
214852024-02-27T21:52:23.933Z 27 540-559 191 191 191 340 340 340 F F F
214862024-02-27T21:52:23.933Z 28 560-579 197 197 197 351 351 351 F F F
214872024-02-27T21:52:23.933Z 29 580-599 194 194 194 345 345 345 F F F
214882024-02-27T21:52:23.933ZMax gen: 199, Max flush: 358
214892024-02-27T21:52:23.933Z8:29 Test duration
214902024-02-27T21:52:23.933ZTest completed
214912024-02-27T21:52:23.937Z
214922024-02-27T21:52:23.937Zreal 8:28.887383366
214932024-02-27T21:52:23.938Zuser 1:35.727128563
214942024-02-27T21:52:23.938Zsys 18.112723539
214952024-02-27T21:52:23.938Ztrap 0.085971270
214962024-02-27T21:52:23.938Ztflt 0.059164695
214972024-02-27T21:52:23.938Zdflt 0.047797299
214982024-02-27T21:52:23.938Zkflt 0.003950721
214992024-02-27T21:52:23.938Zlock 9:54:31.184939174
215002024-02-27T21:52:23.938Zslp 40:28.567885895
215012024-02-27T21:52:23.938Zlat 6.568969916
215022024-02-27T21:52:23.938Zstop 0.049309908
215032024-02-27T21:52:23.938Zprocess exited: duration 517809 ms, exit code 0
 
215042024-02-27T21:52:23.949Zfound 4 output files
215052024-02-27T21:52:23.949Zuploading: /tmp/test_repair_ds8810.txt (47544 bytes)
215062024-02-27T21:52:24.968Zuploaded: /tmp/test_repair_ds8810.txt
215072024-02-27T21:52:24.973Zuploading: /tmp/test_repair_ds8820.txt (192890 bytes)
215082024-02-27T21:52:25.987Zuploaded: /tmp/test_repair_ds8820.txt
215092024-02-27T21:52:25.990Zuploading: /tmp/test_repair_ds8830.txt (20595 bytes)
215102024-02-27T21:52:27.000Zuploaded: /tmp/test_repair_ds8830.txt
215112024-02-27T21:52:27.003Zuploading: /tmp/test_repair_out.txt (77211 bytes)
215122024-02-27T21:52:28.015Zuploaded: /tmp/test_repair_out.txt