Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.30218148 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.049383167 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.049238704 Region set with 3 regions was created /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.113395749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.114036627Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.114065067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188} {"msg":"Crucible 526f2ad7-2c69-4e0b-9cd9-142796fb3f0c has session id: 7ac68cbf-c10c-47ab-be70-dfada97be8bf","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.114475242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Upstairs opts: Upstairs UUID: 526f2ad7-2c69-4e0b-9cd9-142796fb3f0c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.114681605Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Crucible stats registered with UUID: 526f2ad7-2c69-4e0b-9cd9-142796fb3f0c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.114712705Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} Crucible runtime is spawned {"msg":"526f2ad7-2c69-4e0b-9cd9-142796fb3f0c active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.115747369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.115809289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.11583665Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.11586158Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.11588417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.11590744Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.115930101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.115957361Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.115980801Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116169384Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116200834Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:34.116290256Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:34.116328726Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:34.116355816Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:34.116415077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:34.116453858Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:34.116483578Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:34.116514028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.11665571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116681621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:34.116701501Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116714431Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116727731Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:34.116742691Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116755331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116768531Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116782072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116796452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:34.116810512Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.117834435Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.117953617Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.118020787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:51959)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.118471083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:47967)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.118560274Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:34361)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.118623095Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.120532569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.120559Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.12058099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.12059517Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.12060867Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.12062192Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.121476071Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.121552472Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.121649124Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122111159Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.12214847Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122240561Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122258511Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122272121Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122284421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122302302Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122316352Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122330152Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122344582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122358433Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122372083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122386473Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122400433Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122414153Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122428203Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122441243Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122454504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122467564Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122546515Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"":"downstairs","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122567185Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122580455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"Wait for a query_work_queue command to finish before sending IO 0","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122597736Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"1","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122611206Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"client":"2","":"downstairs","session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122624706Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122637066Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"526f2ad7-2c69-4e0b-9cd9-142796fb3f0c is now active with session: 7ac68cbf-c10c-47ab-be70-dfada97be8bf","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122649936Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122662036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188,"session_id":"7ac68cbf-c10c-47ab-be70-dfada97be8bf"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:44.122674186Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 6 Acked 1005 Read 88 Done Done Done false 7 Acked 1006 Read 256 Done Done Done false 8 Acked 1007 Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 8 7 6 5 4 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 9 8 7 6 5 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:48.204702458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1188} Begin loop 1 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.233386142Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.233677185Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.233698095Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190} {"msg":"Crucible 771a82fe-29c9-4381-8e37-9bc59c6c79c5 has session id: e93abb97-f096-470b-bffe-c9d9df6ebdab","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234261893Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Upstairs opts: Upstairs UUID: 771a82fe-29c9-4381-8e37-9bc59c6c79c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234307533Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Crucible stats registered with UUID: 771a82fe-29c9-4381-8e37-9bc59c6c79c5","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234329763Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234639927Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190} {"msg":"771a82fe-29c9-4381-8e37-9bc59c6c79c5 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234665068Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234680248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234694448Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234708248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234722159Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234735449Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.234749169Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.23482286Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.23484549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.23486025Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:48.235044753Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_connection connected from Ok(127.0.0.1:56258)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.235082483Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_connection connected from Ok(127.0.0.1:58037)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.235107733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:48.235135574Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:48.235249215Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.235271775Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.235287586Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.235386237Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.235933264Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.235960224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.235975044Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.236032195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.236584402Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.236615443Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.237166999Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.23719473Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.237219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.23723674Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.237250991Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:48.237264451Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.236048395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_connection connected from Ok(127.0.0.1:48453)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.236243218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.237636405Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.237678886Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.238248143Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.23877484Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.23879726Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[0]R flush_numbers[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.238885761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.238908401Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.238923622Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[1]R flush_numbers[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.238937532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.238951042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.238969022Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[2]R flush_numbers[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.238983582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.238997442Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239011943Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239025013Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239038463Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Next flush: 3","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239051483Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239064413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"":"downstairs","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239077803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239089694Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"0","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239103074Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"1","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239118564Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"client":"2","":"downstairs","session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239132194Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239144554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"771a82fe-29c9-4381-8e37-9bc59c6c79c5 is now active with session: e93abb97-f096-470b-bffe-c9d9df6ebdab","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239157544Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239169615Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190,"session_id":"e93abb97-f096-470b-bffe-c9d9df6ebdab"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:26:58.239181815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 383 len 16384 data: 2 2 2 2 02/30 Read block 256 len 20480 03/30 Read block 287 len 40960 04/30 Read block 145 len 16384 05/30 Write block 485 len 28672 data: 2 2 2 2 2 2 2 06/30 Read block 86 len 20480 07/30 Write block 493 len 24576 data: 2 2 2 2 2 2 08/30 Read block 466 len 12288 09/30 Flush 10/30 Flush 11/30 Write block 28 len 8192 data: 2 2 12/30 Flush 13/30 Write block 135 len 20480 data: 2 2 2 2 2 14/30 Read block 429 len 32768 15/30 Read block 211 len 32768 16/30 Read block 441 len 8192 17/30 Read block 125 len 12288 18/30 Read block 400 len 16384 19/30 Read block 437 len 12288 20/30 Write block 317 len 36864 data: 2 2 2 2 2 2 2 2 2 21/30 Write block 369 len 16384 data: 2 2 2 2 22/30 Write block 406 len 16384 data: 2 2 2 2 23/30 Read block 377 len 40960 24/30 Read block 138 len 12288 25/30 Read block 161 len 20480 26/30 Write block 229 len 28672 data: 2 2 2 2 2 2 2 27/30 Write block 190 len 12288 data: 2 2 2 28/30 Read block 452 len 8192 29/30 Read block 188 len 20480 30/30 Write block 368 len 28672 data: 2 3 3 3 3 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Read 3 Done Done Done false 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:26:58.266451631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 1 1 1 2 2 2 F F F 4 080-099 1 1 1 2 2 2 F F F 5 100-119 1 1 1 2 2 2 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 1 1 1 2 2 2 F F F 8 160-179 1 1 1 2 2 2 F F F 9 180-199 1 1 1 2 2 2 T T T 10 200-219 1 1 1 2 2 2 F F F 11 220-239 1 1 1 2 2 2 T T T 12 240-259 1 1 1 2 2 2 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 T T T 16 320-339 1 1 1 1 1 1 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 T T T 19 380-399 2 2 2 3 3 3 F F F 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 3 3 3 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 6 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 2 2 2 5 5 5 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 1 1 1 2 2 2 F F F 4 080-099 1 1 1 2 2 2 F F F 5 100-119 1 1 1 2 2 2 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 1 1 1 2 2 2 F F F 8 160-179 1 1 1 2 2 2 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 1 1 1 2 2 2 F F F 11 220-239 3 3 3 7 7 7 F F F 12 240-259 1 1 1 2 2 2 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 7 7 7 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 7 7 7 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 3 3 3 7 7 7 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 3 3 3 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 7 Begin loop 2 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.508917618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.509229792Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.509253422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198} {"msg":"Crucible 165103e3-5df9-4323-a335-b6195b85591e has session id: 3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.509641687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Upstairs opts: Upstairs UUID: 165103e3-5df9-4323-a335-b6195b85591e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.509686368Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Crucible stats registered with UUID: 165103e3-5df9-4323-a335-b6195b85591e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.509709818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510059813Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198} {"msg":"165103e3-5df9-4323-a335-b6195b85591e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510128923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510146903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510161574Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510175404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510188804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510203504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510226344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510245675Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510266965Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510333606Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:49703)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510453767Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:41108)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510489408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:27:12.51066224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:27:12.510704701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:27:12.510799572Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510832292Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510857852Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.510975974Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.511308558Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.511331689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.511345969Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.511359529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.511999047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.512030118Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.512600495Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.512627225Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.512641995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.512654545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.512667995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:12.512681226Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.511337989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39261)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.511548631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.512645645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.512680346Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.513252333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.51385147Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.513876631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[0]R flush_numbers[0..12]: [2, 5, 2, 2, 2, 2, 6, 2, 2, 7, 2, 7]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.513894761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[0]R generation[0..12]: [1, 2, 1, 1, 1, 1, 2, 1, 1, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.513917531Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.513936282Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[1]R flush_numbers[0..12]: [2, 5, 2, 2, 2, 2, 6, 2, 2, 7, 2, 7]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.513953662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[1]R generation[0..12]: [1, 2, 1, 1, 1, 1, 2, 1, 1, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.513970522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.513988242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[2]R flush_numbers[0..12]: [2, 5, 2, 2, 2, 2, 6, 2, 2, 7, 2, 7]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514005183Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[2]R generation[0..12]: [1, 2, 1, 1, 1, 1, 2, 1, 1, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514026153Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514043803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514060073Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514076533Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514092784Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514109034Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"":"downstairs","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514125624Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514140734Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"0","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514157494Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"1","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514176665Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"client":"2","":"downstairs","session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514193545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514208785Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"165103e3-5df9-4323-a335-b6195b85591e is now active with session: 3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514224885Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514240226Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:22.514253656Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198,"session_id":"3e29aba3-317b-4e86-9a4e-ed5c2f61b6e1"} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 147 len 8192 data: 2 2 02/30 Read block 10 len 12288 03/30 Write block 466 len 24576 data: 2 2 2 2 2 2 04/30 Write block 506 len 12288 data: 2 2 2 05/30 Read block 420 len 20480 06/30 Write block 395 len 32768 data: 2 2 2 2 2 2 2 2 07/30 Read block 532 len 4096 08/30 Read block 16 len 4096 09/30 Write block 407 len 28672 data: 3 3 3 2 2 2 2 10/30 Read block 560 len 36864 11/30 Read block 328 len 24576 12/30 Write block 424 len 36864 data: 2 2 2 2 2 2 2 2 2 13/30 Read block 189 len 16384 14/30 Flush 15/30 Write block 484 len 32768 data: 2 3 3 3 3 3 3 3 16/30 Read block 216 len 36864 17/30 Read block 280 len 8192 18/30 Read block 171 len 4096 19/30 Write block 244 len 20480 data: 2 2 2 2 2 20/30 Flush 21/30 Write block 21 len 20480 data: 2 2 2 2 2 22/30 Write block 95 len 40960 data: 2 2 2 2 2 2 2 2 2 2 23/30 Read block 451 len 28672 24/30 Write block 51 len 12288 data: 2 2 2 25/30 Write block 557 len 4096 data: 2 26/30 Write block 210 len 16384 data: 2 2 2 2 27/30 Write block 447 len 8192 data: 2 2 28/30 Write block 52 len 20480 data: 3 3 2 2 2 29/30 Write block 334 len 20480 data: 2 2 2 2 2 30/30 Write block 321 len 40960 data: 3 3 3 3 3 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 5 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:27:22.538395712Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 2 2 2 5 5 5 T T T 2 040-059 1 1 1 2 2 2 T T T 3 060-079 1 1 1 2 2 2 F F F 4 080-099 1 1 1 2 2 2 T T T 5 100-119 1 1 1 2 2 2 T T T 6 120-139 2 2 2 6 6 6 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 1 1 1 2 2 2 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 1 1 1 2 2 2 T T T 11 220-239 3 3 3 7 7 7 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 7 7 7 F F F 16 320-339 3 3 3 7 7 7 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 7 7 7 F F F 19 380-399 4 4 4 8 8 8 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 4 4 4 8 8 8 F F F 22 440-459 1 1 1 1 1 1 T T T 23 460-479 4 4 4 8 8 8 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 9 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 5 5 5 10 10 10 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 1 1 1 2 2 2 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 5 5 5 10 10 10 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 1 1 1 2 2 2 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 3 3 3 7 7 7 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 7 7 7 F F F 16 320-339 5 5 5 10 10 10 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 7 7 7 F F F 19 380-399 4 4 4 8 8 8 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 4 4 4 8 8 8 F F F 22 440-459 5 5 5 10 10 10 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 5 5 5 10 10 10 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.806449134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.806747157Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.806763438Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206} {"msg":"Crucible d534a331-24f7-48fd-86e1-8651bcb07d75 has session id: 88deb1bd-c255-41cb-89a2-8c90ebd720e7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807282404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Upstairs opts: Upstairs UUID: d534a331-24f7-48fd-86e1-8651bcb07d75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807330885Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Crucible stats registered with UUID: d534a331-24f7-48fd-86e1-8651bcb07d75","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807358875Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807631169Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206} {"msg":"d534a331-24f7-48fd-86e1-8651bcb07d75 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807678079Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807695339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807710299Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.80772423Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.80774566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.80776998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807796231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807822051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807845631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807860961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61695)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.807989003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56019)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.808050264Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:27:36.808208716Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:27:36.808249376Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:27:36.808326567Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.808344758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.808358968Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.808484359Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.809014476Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.809041096Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.809057156Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.809080607Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.809567553Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.809678975Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.810002688Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.810030929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.810048559Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.810176861Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.810198601Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:36.810215311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.809306Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:64251)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.809454952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.810527065Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.810573896Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811193684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.81173077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811750001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[0]R flush_numbers[0..12]: [2, 10, 10, 2, 10, 10, 6, 8, 2, 7, 10, 7]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811779411Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[0]R generation[0..12]: [1, 5, 5, 1, 5, 5, 2, 4, 1, 3, 5, 3]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811798221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811812862Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[1]R flush_numbers[0..12]: [2, 10, 10, 2, 10, 10, 6, 8, 2, 7, 10, 7]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811826802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[1]R generation[0..12]: [1, 5, 5, 1, 5, 5, 2, 4, 1, 3, 5, 3]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811840442Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811854612Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[2]R flush_numbers[0..12]: [2, 10, 10, 2, 10, 10, 6, 8, 2, 7, 10, 7]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811868242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[2]R generation[0..12]: [1, 5, 5, 1, 5, 5, 2, 4, 1, 3, 5, 3]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811881782Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811896052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811909193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811922393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811935323Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811952153Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"":"downstairs","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811965504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811977584Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"0","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.811993194Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"1","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.812006474Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"client":"2","":"downstairs","session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.812019854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.812032204Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"d534a331-24f7-48fd-86e1-8651bcb07d75 is now active with session: 88deb1bd-c255-41cb-89a2-8c90ebd720e7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.812045205Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.812057134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206,"session_id":"88deb1bd-c255-41cb-89a2-8c90ebd720e7"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:27:46.812069365Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 20480 data: 3 3 3 2 2 02/30 Read block 276 len 24576 03/30 Read block 418 len 24576 04/30 Write block 373 len 40960 data: 3 3 2 2 2 2 2 2 2 2 05/30 Write block 590 len 12288 data: 2 2 2 06/30 Write block 221 len 36864 data: 2 2 2 2 2 2 2 2 3 07/30 Read block 12 len 24576 08/30 Read block 138 len 36864 09/30 Read block 406 len 8192 10/30 Flush 11/30 Write block 526 len 24576 data: 2 2 2 2 2 2 12/30 Write block 418 len 24576 data: 2 2 2 2 2 2 13/30 Read block 532 len 4096 14/30 Read block 508 len 28672 15/30 Write block 48 len 28672 data: 2 2 2 3 4 4 3 16/30 Write block 151 len 24576 data: 2 2 2 2 2 2 17/30 Read block 129 len 12288 18/30 Read block 118 len 4096 19/30 Read block 26 len 40960 20/30 Read block 506 len 32768 21/30 Read block 225 len 40960 22/30 Flush 23/30 Write block 19 len 16384 data: 2 2 3 3 24/30 Read block 179 len 36864 25/30 Write block 353 len 28672 data: 2 2 2 2 2 2 2 26/30 Read block 508 len 12288 27/30 Write block 16 len 4096 data: 2 28/30 Read block 414 len 16384 29/30 Read block 14 len 20480 30/30 Read block 168 len 8192 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:27:46.839010057Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 T T T 1 020-039 5 5 5 10 10 10 T T T 2 040-059 6 6 6 12 12 12 F F F 3 060-079 1 1 1 2 2 2 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 5 5 5 10 10 10 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 6 6 6 12 12 12 F F F 8 160-179 1 1 1 2 2 2 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 7 7 7 F F F 16 320-339 5 5 5 10 10 10 F F F 17 340-359 1 1 1 1 1 1 T T T 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 6 6 6 12 12 12 F F F 21 420-439 6 6 6 12 12 12 F F F 22 440-459 5 5 5 10 10 10 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 6 6 6 12 12 12 F F F 27 540-559 5 5 5 10 10 10 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 6 6 6 11 11 11 F F F Max gen: 6, Max flush: 12 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 7 7 7 13 13 13 F F F 2 040-059 6 6 6 12 12 12 F F F 3 060-079 1 1 1 2 2 2 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 5 5 5 10 10 10 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 6 6 6 12 12 12 F F F 8 160-179 1 1 1 2 2 2 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 6 6 6 11 11 11 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 7 7 7 F F F 16 320-339 5 5 5 10 10 10 F F F 17 340-359 7 7 7 13 13 13 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 6 6 6 12 12 12 F F F 21 420-439 6 6 6 12 12 12 F F F 22 440-459 5 5 5 10 10 10 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 6 6 6 12 12 12 F F F 27 540-559 5 5 5 10 10 10 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 6 6 6 11 11 11 F F F Max gen: 7, Max flush: 13 Begin loop 4 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.040167529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.040531094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.040567064Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214} {"msg":"Crucible a03e4487-2ca7-4dad-bfc3-0d143809f989 has session id: 0135c28a-a315-496e-8628-e40dcaa5f961","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.040907699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Upstairs opts: Upstairs UUID: a03e4487-2ca7-4dad-bfc3-0d143809f989, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041153212Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Crucible stats registered with UUID: a03e4487-2ca7-4dad-bfc3-0d143809f989","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041189922Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041524177Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214} {"msg":"a03e4487-2ca7-4dad-bfc3-0d143809f989 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041548317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041565987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041590887Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041617728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041643408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041668178Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041693819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041719129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041744949Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.04177571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_connection connected from Ok(127.0.0.1:38452)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.041803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:01.041993412Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:01.042033583Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:01.042063923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.042087944Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_connection connected from Ok(127.0.0.1:53018)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.042114264Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.042142254Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.042169005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.042751052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.042776353Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.042790733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.042808173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.043415701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.043538952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.043920507Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.043940367Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.043954217Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.044039749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.044060479Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:01.044074519Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.043086726Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_connection connected from Ok(127.0.0.1:36098)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.043241028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.044454394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.044487604Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045033531Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045557428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045659529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[0]R flush_numbers[0..12]: [13, 13, 12, 2, 10, 10, 6, 12, 2, 7, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045696419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[0]R generation[0..12]: [7, 7, 6, 1, 5, 5, 2, 6, 1, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.04572125Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.04574021Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[1]R flush_numbers[0..12]: [13, 13, 12, 2, 10, 10, 6, 12, 2, 7, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.04575897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[1]R generation[0..12]: [7, 7, 6, 1, 5, 5, 2, 6, 1, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.04577316Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045787611Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[2]R flush_numbers[0..12]: [13, 13, 12, 2, 10, 10, 6, 12, 2, 7, 10, 11]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045801701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"[2]R generation[0..12]: [7, 7, 6, 1, 5, 5, 2, 6, 1, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045815481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045833591Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045847001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045860472Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045873632Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045886802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"":"downstairs","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045900202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045912322Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"0","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045925702Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"1","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045942022Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"client":"2","":"downstairs","session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045955933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045968353Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"a03e4487-2ca7-4dad-bfc3-0d143809f989 is now active with session: 0135c28a-a315-496e-8628-e40dcaa5f961","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.045981523Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.046008154Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214,"session_id":"0135c28a-a315-496e-8628-e40dcaa5f961"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:11.046023554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 36864 data: 3 3 3 3 3 2 2 2 2 02/30 Read block 465 len 24576 03/30 Read block 414 len 8192 04/30 Flush 05/30 Write block 561 len 20480 data: 2 2 2 2 2 06/30 Read block 87 len 32768 07/30 Read block 40 len 12288 08/30 Read block 584 len 4096 09/30 Read block 64 len 36864 10/30 Write block 221 len 40960 data: 3 3 3 3 3 3 3 3 4 3 11/30 Read block 289 len 28672 12/30 Read block 513 len 28672 13/30 Read block 307 len 8192 14/30 Read block 445 len 24576 15/30 Write block 325 len 28672 data: 4 3 3 3 3 3 2 16/30 Write block 207 len 24576 data: 2 2 2 3 3 3 17/30 Write block 340 len 36864 data: 2 2 2 2 2 2 2 2 2 18/30 Flush 19/30 Write block 88 len 32768 data: 2 2 2 2 2 2 2 3 20/30 Write block 543 len 40960 data: 2 2 2 2 2 2 2 2 2 2 21/30 Read block 476 len 24576 22/30 Write block 91 len 16384 data: 3 3 3 3 23/30 Write block 29 len 12288 data: 3 2 2 24/30 Read block 238 len 28672 25/30 Read block 114 len 24576 26/30 Read block 116 len 8192 27/30 Read block 553 len 16384 28/30 Write block 94 len 28672 data: 4 4 3 3 3 3 3 29/30 Read block 569 len 28672 30/30 Read block 458 len 12288 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Write 10 Done Done Done false 21 Acked 1020 Read 6 Done Done Done false 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:11.07409706Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 7 7 7 13 13 13 T T T 2 040-059 6 6 6 12 12 12 F F F 3 060-079 1 1 1 2 2 2 F F F 4 080-099 5 5 5 10 10 10 T T T 5 100-119 5 5 5 10 10 10 T T T 6 120-139 2 2 2 6 6 6 F F F 7 140-159 8 8 8 14 14 14 F F F 8 160-179 8 8 8 14 14 14 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 8 8 8 15 15 15 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 7 7 7 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 8 8 8 15 15 15 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 6 6 6 12 12 12 F F F 21 420-439 6 6 6 12 12 12 F F F 22 440-459 5 5 5 10 10 10 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 6 6 6 12 12 12 F F F 27 540-559 5 5 5 10 10 10 T T T 28 560-579 8 8 8 15 15 15 F F F 29 580-599 6 6 6 11 11 11 F F F Max gen: 8, Max flush: 15 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 F F F 1 020-039 9 9 9 16 16 16 F F F 2 040-059 6 6 6 12 12 12 F F F 3 060-079 1 1 1 2 2 2 F F F 4 080-099 9 9 9 16 16 16 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 8 8 8 14 14 14 F F F 8 160-179 8 8 8 14 14 14 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 8 8 8 15 15 15 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 7 7 7 F F F 16 320-339 8 8 8 15 15 15 F F F 17 340-359 8 8 8 15 15 15 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 6 6 6 12 12 12 F F F 21 420-439 6 6 6 12 12 12 F F F 22 440-459 5 5 5 10 10 10 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 6 6 6 12 12 12 F F F 27 540-559 9 9 9 16 16 16 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 6 6 6 11 11 11 F F F Max gen: 9, Max flush: 16 Begin loop 5 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.285482682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.285735226Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.285761256Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222} {"msg":"Crucible 45c9afdf-f11a-4c34-b0c8-6160323834ba has session id: 294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286151961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Upstairs opts: Upstairs UUID: 45c9afdf-f11a-4c34-b0c8-6160323834ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286208202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Crucible stats registered with UUID: 45c9afdf-f11a-4c34-b0c8-6160323834ba","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286233142Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286498755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222} {"msg":"45c9afdf-f11a-4c34-b0c8-6160323834ba active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286529446Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286546636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286564426Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286581556Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286602997Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286631747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286662737Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286691768Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286715778Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.286742209Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:46535)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.28689399Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:34084)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.287058413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:25.287101063Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:25.287118583Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:25.287134594Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.287147043Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.287160364Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.287291785Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.287770642Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.287805022Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.287831282Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.287897403Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.288385279Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.288499281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.288832845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.288871045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.288900306Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.288926506Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.288953737Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:25.288975767Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.288309408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:51611)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.28840327Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.289711756Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.289739927Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.290237323Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.29077362Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.29079715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[0]R flush_numbers[0..12]: [13, 16, 12, 2, 16, 16, 6, 14, 14, 7, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.29081497Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[0]R generation[0..12]: [7, 9, 6, 1, 9, 9, 2, 8, 8, 3, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.29083735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.290855911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[1]R flush_numbers[0..12]: [13, 16, 12, 2, 16, 16, 6, 14, 14, 7, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.290873771Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[1]R generation[0..12]: [7, 9, 6, 1, 9, 9, 2, 8, 8, 3, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.290890781Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.290908841Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[2]R flush_numbers[0..12]: [13, 16, 12, 2, 16, 16, 6, 14, 14, 7, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.290925982Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[2]R generation[0..12]: [7, 9, 6, 1, 9, 9, 2, 8, 8, 3, 8, 8]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:28:35.290943062Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.290961932Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.290978272Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.290996183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291023973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291042143Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"":"downstairs","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291059183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291074263Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"0","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291094684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"1","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291111684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"client":"2","":"downstairs","session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291129014Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291144534Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"45c9afdf-f11a-4c34-b0c8-6160323834ba is now active with session: 294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291160645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291175775Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222,"session_id":"294f7bbf-c9e9-494b-b0e2-ba9bcd47f1ce"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:35.291192335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 222 len 4096 data: 4 03/30 Read block 508 len 40960 04/30 Write block 314 len 36864 data: 2 2 2 3 3 3 3 4 4 05/30 Write block 473 len 24576 data: 2 2 2 2 2 2 06/30 Flush 07/30 Write block 590 len 36864 data: 3 3 3 2 2 2 2 2 2 08/30 Read block 545 len 8192 09/30 Write block 65 len 4096 data: 2 10/30 Read block 53 len 36864 11/30 Read block 222 len 4096 12/30 Read block 423 len 32768 13/30 Read block 217 len 24576 14/30 Write block 226 len 8192 data: 4 4 15/30 Write block 270 len 4096 data: 2 16/30 Read block 456 len 36864 17/30 Write block 441 len 40960 data: 2 2 2 2 2 2 3 3 2 2 18/30 Flush 19/30 Write block 40 len 12288 data: 2 2 2 20/30 Read block 268 len 12288 21/30 Write block 233 len 16384 data: 3 3 3 2 22/30 Write block 11 len 16384 data: 2 2 2 2 23/30 Read block 60 len 28672 24/30 Read block 157 len 24576 25/30 Write block 382 len 4096 data: 3 26/30 Read block 181 len 16384 27/30 Read block 21 len 40960 28/30 Read block 112 len 36864 29/30 Read block 254 len 40960 30/30 Read block 39 len 28672 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Read 3 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 0 1 Done 11 12 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:35.317186305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 13 13 13 T T T 1 020-039 9 9 9 16 16 16 F F F 2 040-059 6 6 6 12 12 12 T T T 3 060-079 10 10 10 19 19 19 F F F 4 080-099 9 9 9 16 16 16 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 8 8 8 14 14 14 F F F 8 160-179 8 8 8 14 14 14 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 10 10 10 19 19 19 T T T 12 240-259 4 4 4 9 9 9 F F F 13 260-279 10 10 10 19 19 19 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 10 10 10 18 18 18 F F F 16 320-339 10 10 10 18 18 18 F F F 17 340-359 8 8 8 15 15 15 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 T T T 20 400-419 6 6 6 12 12 12 F F F 21 420-439 6 6 6 12 12 12 F F F 22 440-459 10 10 10 19 19 19 F F F 23 460-479 10 10 10 18 18 18 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 6 6 6 12 12 12 F F F 27 540-559 9 9 9 16 16 16 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 10, Max flush: 19 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 20 20 20 F F F 1 020-039 9 9 9 16 16 16 F F F 2 040-059 11 11 11 20 20 20 F F F 3 060-079 10 10 10 19 19 19 F F F 4 080-099 9 9 9 16 16 16 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 8 8 8 14 14 14 F F F 8 160-179 8 8 8 14 14 14 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 11 11 11 20 20 20 F F F 12 240-259 4 4 4 9 9 9 F F F 13 260-279 10 10 10 19 19 19 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 10 10 10 18 18 18 F F F 16 320-339 10 10 10 18 18 18 F F F 17 340-359 8 8 8 15 15 15 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 11 11 11 20 20 20 F F F 20 400-419 6 6 6 12 12 12 F F F 21 420-439 6 6 6 12 12 12 F F F 22 440-459 10 10 10 19 19 19 F F F 23 460-479 10 10 10 18 18 18 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 6 6 6 12 12 12 F F F 27 540-559 9 9 9 16 16 16 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 11, Max flush: 20 Begin loop 6 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.536929214Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.537337369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.537357569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230} {"msg":"Crucible eb3d7106-4c64-499c-bb11-f91aa5cef3d8 has session id: 61e8d16b-7942-4936-b121-5d8671a3bc7f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.537776804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Upstairs opts: Upstairs UUID: eb3d7106-4c64-499c-bb11-f91aa5cef3d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.537820915Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Crucible stats registered with UUID: eb3d7106-4c64-499c-bb11-f91aa5cef3d8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.537838305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538112919Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230} {"msg":"eb3d7106-4c64-499c-bb11-f91aa5cef3d8 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.53818688Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.53820481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.53821994Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.53823353Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.53825431Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538278461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538303841Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538328611Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538359072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538385532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56717)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538504114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46338)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538769177Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:49.538799627Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:49.538817288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:49.538895998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538913579Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.538928719Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.53903261Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.539277814Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.539297464Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.539402835Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.539429995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.539746659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.539862591Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.540306316Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.540332217Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.540346597Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.540371687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.540385717Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:49.540399358Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.539953862Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51565)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.540114974Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.541508892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.541538202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542051589Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542603466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542624176Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[0]R flush_numbers[0..12]: [20, 16, 20, 19, 16, 16, 6, 14, 14, 7, 15, 20]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542639326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[0]R generation[0..12]: [11, 9, 11, 10, 9, 9, 2, 8, 8, 3, 8, 11]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542659786Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542674937Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[1]R flush_numbers[0..12]: [20, 16, 20, 19, 16, 16, 6, 14, 14, 7, 15, 20]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542688847Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[1]R generation[0..12]: [11, 9, 11, 10, 9, 9, 2, 8, 8, 3, 8, 11]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542702707Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542716787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[2]R flush_numbers[0..12]: [20, 16, 20, 19, 16, 16, 6, 14, 14, 7, 15, 20]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542730557Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[2]R generation[0..12]: [11, 9, 11, 10, 9, 9, 2, 8, 8, 3, 8, 11]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542744167Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542758207Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542771178Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542788468Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542802508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542815638Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"":"downstairs","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542829019Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542841009Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"0","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542857129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"1","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542870609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"client":"2","":"downstairs","session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542884229Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.542896359Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"eb3d7106-4c64-499c-bb11-f91aa5cef3d8 is now active with session: 61e8d16b-7942-4936-b121-5d8671a3bc7f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.54290926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.54292149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230,"session_id":"61e8d16b-7942-4936-b121-5d8671a3bc7f"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:28:59.54293358Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 150 len 40960 data: 2 3 4 4 4 4 4 3 3 3 02/30 Write block 8 len 12288 data: 2 2 2 03/30 Write block 299 len 20480 data: 2 2 2 2 2 04/30 Read block 580 len 4096 05/30 Write block 421 len 12288 data: 3 3 3 06/30 Read block 195 len 24576 07/30 Read block 525 len 40960 08/30 Read block 163 len 24576 09/30 Read block 177 len 4096 10/30 Read block 321 len 24576 11/30 Read block 573 len 24576 12/30 Flush 13/30 Write block 461 len 20480 data: 2 2 2 2 2 14/30 Read block 288 len 28672 15/30 Flush 16/30 Write block 316 len 36864 data: 3 4 4 4 4 5 5 4 4 17/30 Read block 491 len 32768 18/30 Read block 49 len 8192 19/30 Read block 370 len 36864 20/30 Read block 53 len 16384 21/30 Write block 143 len 28672 data: 2 2 2 2 3 3 2 22/30 Write block 410 len 36864 data: 3 3 3 3 2 2 2 2 3 23/30 Write block 509 len 4096 data: 2 24/30 Read block 554 len 24576 25/30 Write block 502 len 16384 data: 2 2 2 2 26/30 Write block 255 len 4096 data: 2 27/30 Read block 190 len 4096 28/30 Write block 319 len 20480 data: 5 5 6 6 5 29/30 Read block 9 len 16384 30/30 Read block 46 len 32768 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 5 Done Done Done false 30 Acked 1029 Read 4 Done Done Done false 31 Acked 1030 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:28:59.56896855Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 21 21 21 F F F 1 020-039 9 9 9 16 16 16 F F F 2 040-059 11 11 11 20 20 20 F F F 3 060-079 10 10 10 19 19 19 F F F 4 080-099 9 9 9 16 16 16 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 12 12 12 21 21 21 T T T 8 160-179 8 8 8 14 14 14 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 11 11 11 20 20 20 F F F 12 240-259 4 4 4 9 9 9 T T T 13 260-279 10 10 10 19 19 19 F F F 14 280-299 12 12 12 21 21 21 F F F 15 300-319 12 12 12 23 23 23 T T T 16 320-339 12 12 12 23 23 23 T T T 17 340-359 8 8 8 15 15 15 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 11 11 11 20 20 20 F F F 20 400-419 6 6 6 12 12 12 T T T 21 420-439 12 12 12 21 21 21 F F F 22 440-459 10 10 10 19 19 19 F F F 23 460-479 12 12 12 22 22 22 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 6 6 6 11 11 11 T T T 26 520-539 6 6 6 12 12 12 F F F 27 540-559 9 9 9 16 16 16 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 12, Max flush: 23 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 21 21 21 F F F 1 020-039 9 9 9 16 16 16 F F F 2 040-059 11 11 11 20 20 20 F F F 3 060-079 10 10 10 19 19 19 F F F 4 080-099 9 9 9 16 16 16 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 13 13 13 24 24 24 F F F 8 160-179 8 8 8 14 14 14 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 8 8 8 15 15 15 F F F 11 220-239 11 11 11 20 20 20 F F F 12 240-259 13 13 13 24 24 24 F F F 13 260-279 10 10 10 19 19 19 F F F 14 280-299 12 12 12 21 21 21 F F F 15 300-319 13 13 13 24 24 24 F F F 16 320-339 13 13 13 24 24 24 F F F 17 340-359 8 8 8 15 15 15 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 11 11 11 20 20 20 F F F 20 400-419 13 13 13 24 24 24 F F F 21 420-439 12 12 12 21 21 21 F F F 22 440-459 10 10 10 19 19 19 F F F 23 460-479 12 12 12 22 22 22 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 13 13 13 24 24 24 F F F 26 520-539 6 6 6 12 12 12 F F F 27 540-559 9 9 9 16 16 16 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 13, Max flush: 24 Begin loop 7 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.809092417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.809456542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.809480622Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238} {"msg":"Crucible 65a499e3-4348-468e-b6d4-af04442bb01c has session id: 6a87c41d-59d6-4fdc-bf95-7b5e8c165950","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810003198Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Upstairs opts: Upstairs UUID: 65a499e3-4348-468e-b6d4-af04442bb01c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810049029Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Crucible stats registered with UUID: 65a499e3-4348-468e-b6d4-af04442bb01c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.81007305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810351893Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238} {"msg":"65a499e3-4348-468e-b6d4-af04442bb01c active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810383683Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810406484Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810429224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810451724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810475364Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810500255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810527315Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810554585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810584546Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810608466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_connection connected from Ok(127.0.0.1:35805)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810668057Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:29:13.810701877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:29:13.810731948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:29:13.810759148Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.810778628Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.81087284Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_connection connected from Ok(127.0.0.1:37081)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.8108993Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.811012041Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.811550148Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.811587729Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.811616859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.8116803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.812294918Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.812323348Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.812869325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.812889575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.812903275Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.812915635Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.812929246Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:13.812942556Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.812823694Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_connection connected from Ok(127.0.0.1:40883)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.813454252Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.814125621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.814153981Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.814682868Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815278325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815306496Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[0]R flush_numbers[0..12]: [21, 16, 20, 19, 16, 16, 6, 24, 14, 7, 15, 20]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815325086Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[0]R generation[0..12]: [12, 9, 11, 10, 9, 9, 2, 13, 8, 3, 8, 11]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815348817Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815367427Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[1]R flush_numbers[0..12]: [21, 16, 20, 19, 16, 16, 6, 24, 14, 7, 15, 20]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815384767Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[1]R generation[0..12]: [12, 9, 11, 10, 9, 9, 2, 13, 8, 3, 8, 11]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815404987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815434018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R flush_numbers[0..12]: [21, 16, 20, 19, 16, 16, 6, 24, 14, 7, 15, 20]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815465448Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[2]R generation[0..12]: [12, 9, 11, 10, 9, 9, 2, 13, 8, 3, 8, 11]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815497198Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815527779Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815553799Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815582719Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.8156119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.81564146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"":"downstairs","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.81566013Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815685161Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"0","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815719741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"1","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815750142Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"client":"2","":"downstairs","session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815768872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815793452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"65a499e3-4348-468e-b6d4-af04442bb01c is now active with session: 6a87c41d-59d6-4fdc-bf95-7b5e8c165950","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815822192Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815850393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238,"session_id":"6a87c41d-59d6-4fdc-bf95-7b5e8c165950"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:23.815872053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 436 len 8192 data: 2 2 02/30 Read block 416 len 28672 03/30 Write block 202 len 40960 data: 2 2 2 2 2 3 3 3 4 4 04/30 Write block 56 len 12288 data: 3 2 2 05/30 Write block 526 len 12288 data: 3 3 3 06/30 Write block 501 len 40960 data: 2 3 3 3 3 3 3 3 3 2 07/30 Write block 451 len 40960 data: 2 2 2 2 2 2 2 2 2 2 08/30 Write block 233 len 36864 data: 4 4 4 3 2 2 2 2 2 09/30 Read block 529 len 4096 10/30 Write block 42 len 20480 data: 3 2 2 2 2 11/30 Read block 459 len 20480 12/30 Flush 13/30 Flush 14/30 Write block 259 len 32768 data: 2 2 2 2 2 2 2 2 15/30 Write block 22 len 16384 data: 4 3 3 3 16/30 Flush 17/30 Write block 548 len 16384 data: 3 3 3 3 18/30 Read block 357 len 32768 19/30 Read block 433 len 16384 20/30 Read block 555 len 28672 21/30 Read block 338 len 36864 22/30 Read block 553 len 8192 23/30 Flush 24/30 Write block 247 len 20480 data: 3 3 2 2 2 25/30 Read block 454 len 16384 26/30 Flush 27/30 Write block 405 len 4096 data: 2 28/30 Read block 185 len 36864 29/30 Read block 338 len 20480 30/30 Write block 376 len 24576 data: 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:29:23.845245516Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 21 21 21 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 10 10 10 19 19 19 F F F 4 080-099 9 9 9 16 16 16 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 13 13 13 24 24 24 F F F 8 160-179 8 8 8 14 14 14 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 14 14 14 25 25 25 F F F 11 220-239 14 14 14 25 25 25 F F F 12 240-259 14 14 14 29 29 29 F F F 13 260-279 14 14 14 27 27 27 F F F 14 280-299 12 12 12 21 21 21 F F F 15 300-319 13 13 13 24 24 24 F F F 16 320-339 13 13 13 24 24 24 F F F 17 340-359 8 8 8 15 15 15 F F F 18 360-379 6 6 6 11 11 11 T T T 19 380-399 11 11 11 20 20 20 T T T 20 400-419 13 13 13 24 24 24 T T T 21 420-439 14 14 14 25 25 25 F F F 22 440-459 14 14 14 25 25 25 F F F 23 460-479 14 14 14 25 25 25 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 14 14 14 25 25 25 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 14 14 14 28 28 28 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 14, Max flush: 29 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 21 21 21 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 10 10 10 19 19 19 F F F 4 080-099 9 9 9 16 16 16 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 13 13 13 24 24 24 F F F 8 160-179 8 8 8 14 14 14 F F F 9 180-199 3 3 3 7 7 7 F F F 10 200-219 14 14 14 25 25 25 F F F 11 220-239 14 14 14 25 25 25 F F F 12 240-259 14 14 14 29 29 29 F F F 13 260-279 14 14 14 27 27 27 F F F 14 280-299 12 12 12 21 21 21 F F F 15 300-319 13 13 13 24 24 24 F F F 16 320-339 13 13 13 24 24 24 F F F 17 340-359 8 8 8 15 15 15 F F F 18 360-379 15 15 15 30 30 30 F F F 19 380-399 15 15 15 30 30 30 F F F 20 400-419 15 15 15 30 30 30 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 14 14 14 25 25 25 F F F 23 460-479 14 14 14 25 25 25 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 14 14 14 25 25 25 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 14 14 14 28 28 28 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 15, Max flush: 30 Begin loop 8 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.048373834Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.048771959Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.048807479Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246} {"msg":"Crucible 04bcb4d4-6e7d-48a7-a6e2-c85a56aa916e has session id: 4397af18-0e61-46b5-a80d-ed2352ed43fb","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049025212Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Upstairs opts: Upstairs UUID: 04bcb4d4-6e7d-48a7-a6e2-c85a56aa916e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049105593Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Crucible stats registered with UUID: 04bcb4d4-6e7d-48a7-a6e2-c85a56aa916e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049125723Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049462327Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246} {"msg":"04bcb4d4-6e7d-48a7-a6e2-c85a56aa916e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049527078Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049544638Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049559229Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049573139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049586869Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049600309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049613879Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.04968083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.04971188Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049740881Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:29:38.049805582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:29:38.049824422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:29:38.049866742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049888073Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.049912843Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:39873)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.050107216Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:63600)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.050208747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.050238807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.050745633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.050773754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.050789674Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.050803974Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.051437832Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.051469903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.051982759Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.05200924Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.05202395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.05203654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.05205017Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:38.05206344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.050602502Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48917)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.050760454Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.05202068Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.05205475Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.052667098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053280956Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053314706Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[0]R flush_numbers[0..12]: [21, 27, 25, 19, 16, 16, 6, 24, 14, 7, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053334557Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[0]R generation[0..12]: [12, 14, 14, 10, 9, 9, 2, 13, 8, 3, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053352667Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053376257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[1]R flush_numbers[0..12]: [21, 27, 25, 19, 16, 16, 6, 24, 14, 7, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053394257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[1]R generation[0..12]: [12, 14, 14, 10, 9, 9, 2, 13, 8, 3, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053411607Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053429308Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[2]R flush_numbers[0..12]: [21, 27, 25, 19, 16, 16, 6, 24, 14, 7, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053446808Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[2]R generation[0..12]: [12, 14, 14, 10, 9, 9, 2, 13, 8, 3, 14, 14]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-07-10T15:29:48.053463838Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053485258Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053501728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053518459Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053534929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053551519Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"":"downstairs","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053568339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.05358362Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"0","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.05360068Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"1","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.05361966Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"client":"2","":"downstairs","session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.05363679Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053652421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"04bcb4d4-6e7d-48a7-a6e2-c85a56aa916e is now active with session: 4397af18-0e61-46b5-a80d-ed2352ed43fb","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053668791Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053683961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246,"session_id":"4397af18-0e61-46b5-a80d-ed2352ed43fb"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:29:48.053699291Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 143 len 36864 data: 3 3 3 3 4 4 3 3 4 02/30 Read block 240 len 36864 03/30 Read block 410 len 20480 04/30 Write block 374 len 4096 data: 4 05/30 Read block 82 len 16384 06/30 Write block 591 len 4096 data: 4 07/30 Read block 596 len 16384 08/30 Write block 64 len 4096 data: 2 09/30 Read block 25 len 20480 10/30 Read block 410 len 36864 11/30 Read block 357 len 36864 12/30 Flush 13/30 Write block 542 len 12288 data: 2 3 3 14/30 Write block 514 len 8192 data: 2 2 15/30 Write block 549 len 8192 data: 4 4 16/30 Write block 450 len 16384 data: 3 3 3 3 17/30 Read block 446 len 32768 18/30 Write block 69 len 8192 data: 2 2 19/30 Flush 20/30 Write block 89 len 16384 data: 3 3 4 4 21/30 Write block 246 len 40960 data: 3 4 4 3 3 3 2 2 2 3 22/30 Write block 191 len 28672 data: 3 3 2 2 2 2 2 23/30 Write block 228 len 4096 data: 4 24/30 Write block 143 len 8192 data: 4 4 25/30 Write block 502 len 12288 data: 4 4 4 26/30 Flush 27/30 Write block 211 len 28672 data: 5 4 3 2 2 2 2 28/30 Write block 508 len 28672 data: 4 4 3 2 2 2 3 29/30 Read block 468 len 24576 30/30 Read block 170 len 8192 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 7 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:29:48.079000682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 21 21 21 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 16 16 16 32 32 32 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 16 16 16 34 34 34 F F F 8 160-179 8 8 8 14 14 14 F F F 9 180-199 16 16 16 34 34 34 F F F 10 200-219 14 14 14 25 25 25 T T T 11 220-239 16 16 16 34 34 34 F F F 12 240-259 16 16 16 34 34 34 F F F 13 260-279 14 14 14 27 27 27 F F F 14 280-299 12 12 12 21 21 21 F F F 15 300-319 13 13 13 24 24 24 F F F 16 320-339 13 13 13 24 24 24 F F F 17 340-359 8 8 8 15 15 15 F F F 18 360-379 16 16 16 31 31 31 F F F 19 380-399 15 15 15 30 30 30 F F F 20 400-419 15 15 15 30 30 30 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 16 16 16 32 32 32 F F F 23 460-479 14 14 14 25 25 25 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 16 16 16 34 34 34 T T T 26 520-539 14 14 14 25 25 25 F F F 27 540-559 16 16 16 32 32 32 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 16, Max flush: 34 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 21 21 21 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 16 16 16 32 32 32 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 9 9 9 16 16 16 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 16 16 16 34 34 34 F F F 8 160-179 8 8 8 14 14 14 F F F 9 180-199 16 16 16 34 34 34 F F F 10 200-219 17 17 17 35 35 35 F F F 11 220-239 16 16 16 34 34 34 F F F 12 240-259 16 16 16 34 34 34 F F F 13 260-279 14 14 14 27 27 27 F F F 14 280-299 12 12 12 21 21 21 F F F 15 300-319 13 13 13 24 24 24 F F F 16 320-339 13 13 13 24 24 24 F F F 17 340-359 8 8 8 15 15 15 F F F 18 360-379 16 16 16 31 31 31 F F F 19 380-399 15 15 15 30 30 30 F F F 20 400-419 15 15 15 30 30 30 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 16 16 16 32 32 32 F F F 23 460-479 14 14 14 25 25 25 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 17 17 17 35 35 35 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 16 16 16 32 32 32 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 17, Max flush: 35 Begin loop 9 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.263761797Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264115601Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264150962Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254} {"msg":"Crucible a74ace04-db24-4d38-9dd8-0963f61c8851 has session id: 6cc2b06e-2731-4119-9841-c0ebb150bf36","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264414535Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Upstairs opts: Upstairs UUID: a74ace04-db24-4d38-9dd8-0963f61c8851, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264460466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Crucible stats registered with UUID: a74ace04-db24-4d38-9dd8-0963f61c8851","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264484516Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.26474786Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254} {"msg":"a74ace04-db24-4d38-9dd8-0963f61c8851 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.26478648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.26480158Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.26481623Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.26483019Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264843941Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264857381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264875131Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264964802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.264992933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.265008763Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:02.265154675Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_connection connected from Ok(127.0.0.1:33086)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.265191725Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_connection connected from Ok(127.0.0.1:65137)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.265219186Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:02.265247766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:02.265277706Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.265301836Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.265326407Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.26556823Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.266049946Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.266076066Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.266090837Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.266104597Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.266735005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.266763005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.267274112Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.267312882Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.267334802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.267354042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.267383103Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:02.267403433Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.267032749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_connection connected from Ok(127.0.0.1:47232)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.267315532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.268254264Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.268282174Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.268781981Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269343118Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269362668Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[0]R flush_numbers[0..12]: [21, 27, 25, 32, 33, 16, 6, 34, 14, 34, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269382338Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[0]R generation[0..12]: [12, 14, 14, 16, 16, 9, 2, 16, 8, 16, 17, 16]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269396628Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269411199Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[1]R flush_numbers[0..12]: [21, 27, 25, 32, 33, 16, 6, 34, 14, 34, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269430539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[1]R generation[0..12]: [12, 14, 14, 16, 16, 9, 2, 16, 8, 16, 17, 16]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269444899Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269459239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[2]R flush_numbers[0..12]: [21, 27, 25, 32, 33, 16, 6, 34, 14, 34, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.2694733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[2]R generation[0..12]: [12, 14, 14, 16, 16, 9, 2, 16, 8, 16, 17, 16]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.26949124Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.26951678Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.26954705Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269577711Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269601871Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269629931Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"":"downstairs","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269661072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269688062Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"0","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269706523Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"1","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269733763Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"client":"2","":"downstairs","session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269764333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269797813Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"a74ace04-db24-4d38-9dd8-0963f61c8851 is now active with session: 6cc2b06e-2731-4119-9841-c0ebb150bf36","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269816814Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269841224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254,"session_id":"6cc2b06e-2731-4119-9841-c0ebb150bf36"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:12.269868265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 249 len 28672 data: 4 4 4 3 3 3 4 02/30 Write block 103 len 16384 data: 3 3 2 2 03/30 Read block 136 len 32768 04/30 Write block 503 len 36864 data: 5 5 4 4 4 5 5 4 3 05/30 Read block 174 len 40960 06/30 Read block 544 len 4096 07/30 Read block 413 len 32768 08/30 Write block 468 len 12288 data: 3 3 3 09/30 Write block 61 len 40960 data: 2 2 2 3 3 2 2 2 3 3 10/30 Flush 11/30 Write block 219 len 32768 data: 2 2 4 5 4 4 4 5 12/30 Read block 127 len 40960 13/30 Write block 158 len 8192 data: 4 4 14/30 Write block 339 len 24576 data: 2 3 3 3 3 3 15/30 Read block 77 len 28672 16/30 Write block 499 len 24576 data: 3 3 3 5 6 6 17/30 Read block 376 len 40960 18/30 Write block 142 len 12288 data: 2 5 5 19/30 Read block 528 len 8192 20/30 Write block 551 len 20480 data: 4 3 2 2 2 21/30 Write block 1 len 16384 data: 2 2 2 2 22/30 Write block 158 len 36864 data: 5 5 3 2 2 2 2 2 2 23/30 Read block 135 len 40960 24/30 Write block 191 len 28672 data: 4 4 3 3 3 3 3 25/30 Write block 285 len 40960 data: 2 2 2 2 2 2 2 2 2 2 26/30 Read block 342 len 32768 27/30 Read block 134 len 20480 28/30 Read block 397 len 36864 29/30 Write block 259 len 16384 data: 3 3 3 3 30/30 Write block 339 len 16384 data: 3 4 4 4 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 4 Done Done Done false 31 Acked 1030 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:12.297275452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 21 21 21 T T T 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 18 18 18 36 36 36 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 18 18 18 36 36 36 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 18 18 18 37 37 37 T T T 8 160-179 8 8 8 14 14 14 T T T 9 180-199 16 16 16 34 34 34 T T T 10 200-219 18 18 18 37 37 37 F F F 11 220-239 18 18 18 37 37 37 F F F 12 240-259 18 18 18 36 36 36 T T T 13 260-279 14 14 14 27 27 27 T T T 14 280-299 12 12 12 21 21 21 T T T 15 300-319 13 13 13 24 24 24 F F F 16 320-339 18 18 18 37 37 37 T T T 17 340-359 18 18 18 37 37 37 T T T 18 360-379 16 16 16 31 31 31 F F F 19 380-399 15 15 15 30 30 30 F F F 20 400-419 15 15 15 30 30 30 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 16 16 16 32 32 32 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 18 18 18 37 37 37 F F F 25 500-519 18 18 18 37 37 37 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 16 16 16 32 32 32 T T T 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 18, Max flush: 37 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 38 38 38 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 18 18 18 36 36 36 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 18 18 18 36 36 36 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 19 19 19 38 38 38 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 18 18 18 37 37 37 F F F 12 240-259 19 19 19 38 38 38 F F F 13 260-279 19 19 19 38 38 38 F F F 14 280-299 19 19 19 38 38 38 F F F 15 300-319 13 13 13 24 24 24 F F F 16 320-339 19 19 19 38 38 38 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 16 16 16 31 31 31 F F F 19 380-399 15 15 15 30 30 30 F F F 20 400-419 15 15 15 30 30 30 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 16 16 16 32 32 32 F F F 23 460-479 18 18 18 36 36 36 F F F 24 480-499 18 18 18 37 37 37 F F F 25 500-519 18 18 18 37 37 37 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 19 19 19 38 38 38 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 19, Max flush: 38 Begin loop 10 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.590182839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.590554064Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.590587195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262} {"msg":"Crucible 0f19a329-3dee-492d-8be1-6304b921ca7e has session id: 860a4f5d-908c-4fc5-98cc-a5ef75f20214","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.590958889Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Upstairs opts: Upstairs UUID: 0f19a329-3dee-492d-8be1-6304b921ca7e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.59100616Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Crucible stats registered with UUID: 0f19a329-3dee-492d-8be1-6304b921ca7e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.59103056Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591296584Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262} {"msg":"0f19a329-3dee-492d-8be1-6304b921ca7e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591430655Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591448786Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591463596Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591477416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591491356Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591504656Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591526836Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591551987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591566887Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591584407Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_connection connected from Ok(127.0.0.1:51555)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.59177374Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_connection connected from Ok(127.0.0.1:48902)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.591930941Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:26.592080014Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:26.592118194Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:26.592149394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.592174745Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.592198295Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.592283526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.592692281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.592727522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.592751032Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.592772992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.593315679Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.593348759Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.593898086Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.593924397Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.593939457Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.593952077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.593965918Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:26.593979467Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.593497552Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_connection connected from Ok(127.0.0.1:59814)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.593604933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.594484294Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.594518425Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595059991Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595589508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595613408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[0]R flush_numbers[0..12]: [38, 27, 25, 36, 33, 36, 6, 38, 38, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595628788Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[0]R generation[0..12]: [19, 14, 14, 18, 16, 18, 2, 19, 19, 19, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595643249Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595662429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[1]R flush_numbers[0..12]: [38, 27, 25, 36, 33, 36, 6, 38, 38, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595676749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[1]R generation[0..12]: [19, 14, 14, 18, 16, 18, 2, 19, 19, 19, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595690689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.5957052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[2]R flush_numbers[0..12]: [38, 27, 25, 36, 33, 36, 6, 38, 38, 38, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.59571917Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[2]R generation[0..12]: [19, 14, 14, 18, 16, 18, 2, 19, 19, 19, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.59573309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.59574728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.59576371Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595777321Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.59579059Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595803631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"":"downstairs","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595817071Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595829201Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"0","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595842461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"1","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595858871Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"client":"2","":"downstairs","session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595872552Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595884902Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"0f19a329-3dee-492d-8be1-6304b921ca7e is now active with session: 860a4f5d-908c-4fc5-98cc-a5ef75f20214","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595897952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595910002Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262,"session_id":"860a4f5d-908c-4fc5-98cc-a5ef75f20214"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:36.595922222Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 359 len 12288 data: 3 2 2 02/30 Read block 273 len 16384 03/30 Read block 468 len 4096 04/30 Read block 39 len 16384 05/30 Read block 260 len 16384 06/30 Read block 146 len 4096 07/30 Write block 476 len 16384 data: 3 3 3 2 08/30 Flush 09/30 Write block 551 len 32768 data: 5 4 3 3 3 2 3 2 10/30 Read block 592 len 8192 11/30 Read block 378 len 24576 12/30 Read block 463 len 12288 13/30 Write block 293 len 12288 data: 3 3 2 14/30 Write block 339 len 20480 data: 4 5 5 5 4 15/30 Write block 3 len 4096 data: 3 16/30 Read block 390 len 24576 17/30 Read block 581 len 32768 18/30 Flush 19/30 Write block 227 len 12288 data: 5 5 5 20/30 Read block 120 len 4096 21/30 Read block 476 len 4096 22/30 Read block 436 len 4096 23/30 Read block 297 len 4096 24/30 Write block 0 len 20480 data: 2 3 3 4 3 25/30 Write block 494 len 36864 data: 3 3 4 4 4 4 4 4 6 26/30 Read block 0 len 16384 27/30 Write block 549 len 36864 data: 5 5 6 5 4 4 4 3 4 28/30 Write block 557 len 12288 data: 5 3 2 29/30 Write block 476 len 12288 data: 4 4 4 30/30 Read block 343 len 24576 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 3 Done Done Sent false 30 Acked 1029 Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 2 3 Done 11 12 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:36.619854526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 40 40 40 T T T 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 18 18 18 36 36 36 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 18 18 18 36 36 36 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 19 19 19 38 38 38 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 18 18 18 37 37 37 T T T 12 240-259 19 19 19 38 38 38 F F F 13 260-279 19 19 19 38 38 38 F F F 14 280-299 20 20 20 40 40 40 F F F 15 300-319 13 13 13 24 24 24 F F F 16 320-339 20 20 20 40 40 40 F F F 17 340-359 20 20 20 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 15 15 15 30 30 30 F F F 20 400-419 15 15 15 30 30 30 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 16 16 16 32 32 32 F F F 23 460-479 20 20 20 39 39 39 T T T 24 480-499 18 18 18 37 37 37 T T T 25 500-519 18 18 18 37 37 37 T T T 26 520-539 14 14 14 25 25 25 F F F 27 540-559 20 20 20 40 40 40 T T T 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 20, Max flush: 40 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 41 41 41 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 18 18 18 36 36 36 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 18 18 18 36 36 36 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 19 19 19 38 38 38 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 21 21 21 41 41 41 F F F 12 240-259 19 19 19 38 38 38 F F F 13 260-279 19 19 19 38 38 38 F F F 14 280-299 20 20 20 40 40 40 F F F 15 300-319 13 13 13 24 24 24 F F F 16 320-339 20 20 20 40 40 40 F F F 17 340-359 20 20 20 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 15 15 15 30 30 30 F F F 20 400-419 15 15 15 30 30 30 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 16 16 16 32 32 32 F F F 23 460-479 21 21 21 41 41 41 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 21 21 21 41 41 41 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 21 21 21 41 41 41 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 21, Max flush: 41 Begin loop 11 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.858068589Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.858499934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.858518804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270} {"msg":"Crucible 8c34d956-c48d-4cfc-b42d-022459b77753 has session id: 73fb06b0-ba09-41fa-b443-87c45a3b8dfd","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.858864269Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Upstairs opts: Upstairs UUID: 8c34d956-c48d-4cfc-b42d-022459b77753, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.858909249Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Crucible stats registered with UUID: 8c34d956-c48d-4cfc-b42d-022459b77753","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.858926969Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859295284Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270} {"msg":"8c34d956-c48d-4cfc-b42d-022459b77753 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859318425Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859331585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859345955Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859359715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859373415Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859386735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859400246Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859474916Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859510167Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859526507Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:50.859656049Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:50.859693719Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:43348)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859717239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:30:50.85974318Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.85976455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.85978587Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859825591Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:61448)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.859911582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.860453169Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.860480629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.860495909Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.860509709Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.861160848Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.861200758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.861743005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.861773716Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.861795676Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.861814766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.861835926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:30:50.861857377Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.860310847Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:63591)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.860487219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.861598143Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.861629914Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.862196811Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.862814309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.862841099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[0]R flush_numbers[0..12]: [41, 27, 25, 36, 33, 36, 6, 38, 38, 38, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.862859279Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[0]R generation[0..12]: [21, 14, 14, 18, 16, 18, 2, 19, 19, 19, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.86288409Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.86290293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[1]R flush_numbers[0..12]: [41, 27, 25, 36, 33, 36, 6, 38, 38, 38, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.8629207Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[1]R generation[0..12]: [21, 14, 14, 18, 16, 18, 2, 19, 19, 19, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.86293822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.862956061Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[2]R flush_numbers[0..12]: [41, 27, 25, 36, 33, 36, 6, 38, 38, 38, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.862973601Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[2]R generation[0..12]: [21, 14, 14, 18, 16, 18, 2, 19, 19, 19, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.862991151Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863024881Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863047802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863065012Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863081312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863097702Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"":"downstairs","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863114513Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863129663Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"0","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863146683Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"1","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863166253Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"client":"2","":"downstairs","session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863183463Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863198834Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"8c34d956-c48d-4cfc-b42d-022459b77753 is now active with session: 73fb06b0-ba09-41fa-b443-87c45a3b8dfd","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863215084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863230164Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270,"session_id":"73fb06b0-ba09-41fa-b443-87c45a3b8dfd"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:00.863245584Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 396 len 32768 data: 3 3 3 3 3 3 3 2 03/30 Write block 550 len 4096 data: 6 04/30 Write block 115 len 28672 data: 2 2 2 2 2 2 2 05/30 Read block 355 len 8192 06/30 Write block 394 len 8192 data: 2 3 07/30 Write block 255 len 28672 data: 5 2 2 2 4 4 4 08/30 Read block 586 len 32768 09/30 Read block 357 len 20480 10/30 Read block 115 len 12288 11/30 Write block 533 len 24576 data: 2 2 2 2 2 2 12/30 Read block 448 len 12288 13/30 Read block 310 len 28672 14/30 Read block 357 len 20480 15/30 Read block 558 len 28672 16/30 Read block 279 len 16384 17/30 Read block 327 len 28672 18/30 Read block 454 len 8192 19/30 Flush 20/30 Write block 303 len 28672 data: 3 2 2 2 2 2 2 21/30 Write block 2 len 16384 data: 4 5 4 2 22/30 Read block 348 len 40960 23/30 Flush 24/30 Write block 418 len 12288 data: 4 3 3 25/30 Write block 277 len 28672 data: 2 2 2 2 2 2 2 26/30 Write block 439 len 24576 data: 2 2 3 3 3 3 27/30 Write block 549 len 8192 data: 6 7 28/30 Read block 438 len 36864 29/30 Write block 361 len 24576 data: 3 2 2 2 2 2 30/30 Write block 377 len 12288 data: 4 4 4 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 6 Done Done Done false 31 Acked 1030 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:31:00.890345278Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 45 45 45 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 18 18 18 36 36 36 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 22 22 22 43 43 43 F F F 6 120-139 22 22 22 43 43 43 F F F 7 140-159 19 19 19 38 38 38 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 21 21 21 41 41 41 F F F 12 240-259 22 22 22 43 43 43 F F F 13 260-279 22 22 22 43 43 43 T T T 14 280-299 20 20 20 40 40 40 T T T 15 300-319 22 22 22 44 44 44 F F F 16 320-339 20 20 20 40 40 40 F F F 17 340-359 20 20 20 40 40 40 F F F 18 360-379 20 20 20 39 39 39 T T T 19 380-399 22 22 22 43 43 43 F F F 20 400-419 22 22 22 43 43 43 T T T 21 420-439 14 14 14 25 25 25 T T T 22 440-459 16 16 16 32 32 32 T T T 23 460-479 21 21 21 41 41 41 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 21 21 21 41 41 41 F F F 26 520-539 22 22 22 43 43 43 F F F 27 540-559 22 22 22 43 43 43 T T T 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 22, Max flush: 45 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 45 45 45 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 18 18 18 36 36 36 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 22 22 22 43 43 43 F F F 6 120-139 22 22 22 43 43 43 F F F 7 140-159 19 19 19 38 38 38 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 19 19 19 38 38 38 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 21 21 21 41 41 41 F F F 12 240-259 22 22 22 43 43 43 F F F 13 260-279 23 23 23 46 46 46 F F F 14 280-299 23 23 23 46 46 46 F F F 15 300-319 22 22 22 44 44 44 F F F 16 320-339 20 20 20 40 40 40 F F F 17 340-359 20 20 20 40 40 40 F F F 18 360-379 23 23 23 46 46 46 F F F 19 380-399 22 22 22 43 43 43 F F F 20 400-419 23 23 23 46 46 46 F F F 21 420-439 23 23 23 46 46 46 F F F 22 440-459 23 23 23 46 46 46 F F F 23 460-479 21 21 21 41 41 41 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 21 21 21 41 41 41 F F F 26 520-539 22 22 22 43 43 43 F F F 27 540-559 23 23 23 46 46 46 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 23, Max flush: 46 Begin loop 12 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.140105277Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.140723115Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.140755526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278} {"msg":"Crucible f79e2ecb-d0cb-41c4-9850-6b69d58df168 has session id: 6355a694-a6ea-4637-835c-3f4a1af1508d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.140879047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Upstairs opts: Upstairs UUID: f79e2ecb-d0cb-41c4-9850-6b69d58df168, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.140913558Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Crucible stats registered with UUID: f79e2ecb-d0cb-41c4-9850-6b69d58df168","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.140930148Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141329663Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278} {"msg":"f79e2ecb-d0cb-41c4-9850-6b69d58df168 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141365513Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141379314Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141393634Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141407324Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141425734Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141448955Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141474815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141499885Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141525355Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141544166Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:31:15.14190398Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:31:15.141943721Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62022)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.141969611Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:31:15.141994992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.142017612Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.142039442Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52437)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.142063093Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.142084783Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.142625989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.14265227Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.142760381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.142788522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.143153656Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.143265288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.143623912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.143654323Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.143678023Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.143767394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.143785394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:15.143798815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.143565382Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43479)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.143859725Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.144830968Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.144857788Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.145375815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.145922742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.145951032Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[0]R flush_numbers[0..12]: [45, 27, 25, 36, 33, 43, 43, 38, 38, 38, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.145969802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[0]R generation[0..12]: [22, 14, 14, 18, 16, 22, 22, 19, 19, 19, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.145987732Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146022333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[1]R flush_numbers[0..12]: [45, 27, 25, 36, 33, 43, 43, 38, 38, 38, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146043683Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[1]R generation[0..12]: [22, 14, 14, 18, 16, 22, 22, 19, 19, 19, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146060943Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146078804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[2]R flush_numbers[0..12]: [45, 27, 25, 36, 33, 43, 43, 38, 38, 38, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146096074Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R generation[0..12]: [22, 14, 14, 18, 16, 22, 22, 19, 19, 19, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146116794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146134824Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146151134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146167855Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146180695Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146193875Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"":"downstairs","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146207075Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146219085Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"0","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146232246Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"1","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146247886Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"client":"2","":"downstairs","session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146261716Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146274026Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"f79e2ecb-d0cb-41c4-9850-6b69d58df168 is now active with session: 6355a694-a6ea-4637-835c-3f4a1af1508d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146287176Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146299156Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278,"session_id":"6355a694-a6ea-4637-835c-3f4a1af1508d"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:25.146311496Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 350 len 28672 data: 2 2 2 3 3 3 3 02/30 Read block 98 len 32768 03/30 Read block 492 len 36864 04/30 Read block 17 len 16384 05/30 Write block 368 len 12288 data: 3 4 4 06/30 Read block 5 len 36864 07/30 Write block 380 len 12288 data: 4 4 4 08/30 Write block 194 len 8192 data: 4 4 09/30 Flush 10/30 Write block 397 len 16384 data: 4 4 4 4 11/30 Write block 93 len 32768 data: 4 5 5 4 4 4 4 4 12/30 Write block 460 len 8192 data: 3 3 13/30 Write block 72 len 8192 data: 2 2 14/30 Write block 463 len 8192 data: 3 3 15/30 Read block 313 len 16384 16/30 Read block 409 len 32768 17/30 Read block 54 len 40960 18/30 Read block 541 len 20480 19/30 Read block 517 len 12288 20/30 Write block 251 len 40960 data: 5 4 4 4 6 3 3 3 5 5 21/30 Read block 564 len 8192 22/30 Read block 20 len 24576 23/30 Read block 314 len 4096 24/30 Flush 25/30 Write block 17 len 20480 data: 2 2 3 3 4 26/30 Read block 344 len 12288 27/30 Write block 18 len 4096 data: 3 28/30 Write block 439 len 40960 data: 3 3 4 4 4 4 3 3 4 4 29/30 Write block 197 len 8192 data: 4 2 30/30 Read block 359 len 12288 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Write 10 Done Done Done false 30 Acked 1029 Write 2 Done Sent Done false 31 Acked 1030 Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 2 0 2 Done 6 4 6 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:31:25.173695724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 45 45 45 T T T 1 020-039 14 14 14 27 27 27 T T T 2 040-059 14 14 14 25 25 25 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 24 24 24 48 48 48 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 22 22 22 43 43 43 F F F 7 140-159 19 19 19 38 38 38 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 24 24 24 47 47 47 T T T 10 200-219 18 18 18 37 37 37 F F F 11 220-239 21 21 21 41 41 41 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 24 24 24 49 49 49 F F F 14 280-299 23 23 23 46 46 46 F F F 15 300-319 22 22 22 44 44 44 F F F 16 320-339 20 20 20 40 40 40 F F F 17 340-359 24 24 24 47 47 47 F F F 18 360-379 24 24 24 47 47 47 F F F 19 380-399 24 24 24 48 48 48 F F F 20 400-419 24 24 24 48 48 48 F F F 21 420-439 23 23 23 46 46 46 T T T 22 440-459 23 23 23 46 46 46 T T T 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 21 21 21 41 41 41 F F F 26 520-539 22 22 22 43 43 43 F F F 27 540-559 23 23 23 46 46 46 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 24, Max flush: 49 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 50 50 50 F F F 1 020-039 25 25 25 50 50 50 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 24 24 24 48 48 48 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 22 22 22 43 43 43 F F F 7 140-159 19 19 19 38 38 38 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 25 25 25 50 50 50 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 21 21 21 41 41 41 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 24 24 24 49 49 49 F F F 14 280-299 23 23 23 46 46 46 F F F 15 300-319 22 22 22 44 44 44 F F F 16 320-339 20 20 20 40 40 40 F F F 17 340-359 24 24 24 47 47 47 F F F 18 360-379 24 24 24 47 47 47 F F F 19 380-399 24 24 24 48 48 48 F F F 20 400-419 24 24 24 48 48 48 F F F 21 420-439 25 25 25 50 50 50 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 21 21 21 41 41 41 F F F 26 520-539 22 22 22 43 43 43 F F F 27 540-559 23 23 23 46 46 46 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 25, Max flush: 50 Begin loop 13 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.39956227Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400049307Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400089837Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286} {"msg":"Crucible 10302526-c446-42d2-8a70-97de89b14f7a has session id: 8908ee2e-89a1-4472-9ba8-f47ce8a5db8d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400136228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Upstairs opts: Upstairs UUID: 10302526-c446-42d2-8a70-97de89b14f7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400233089Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Crucible stats registered with UUID: 10302526-c446-42d2-8a70-97de89b14f7a","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400260889Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400670004Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286} {"msg":"10302526-c446-42d2-8a70-97de89b14f7a active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400705085Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400732945Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400756145Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400780956Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400805246Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400827566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400851707Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400875887Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400899097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.400923578Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57958)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.401183201Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55540)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.401301083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:31:39.401334333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:31:39.401396404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:31:39.401426344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.401441314Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.401455144Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.401585856Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.402145683Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.402179923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.402207664Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.402234194Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.402783991Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.402922003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.403318708Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.403347878Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.403362918Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.403379459Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.403393399Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:39.403406659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.402743651Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43648)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.402873803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.403974926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.404005537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.404633635Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405182612Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405207302Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[0]R flush_numbers[0..12]: [50, 50, 25, 48, 48, 48, 43, 38, 38, 50, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405225412Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[0]R generation[0..12]: [25, 25, 14, 24, 24, 24, 22, 19, 19, 25, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405242962Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405266193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[1]R flush_numbers[0..12]: [50, 50, 25, 48, 48, 48, 43, 38, 38, 50, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405284173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[1]R generation[0..12]: [25, 25, 14, 24, 24, 24, 22, 19, 19, 25, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405301473Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405319114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[2]R flush_numbers[0..12]: [50, 50, 25, 48, 48, 48, 43, 38, 38, 50, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405336344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [25, 25, 14, 24, 24, 24, 22, 19, 19, 25, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405356614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405375024Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405391404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405407835Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405424115Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405440645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"":"downstairs","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405457625Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405472716Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"0","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405489756Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"1","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405506526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"client":"2","":"downstairs","session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405525696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405541386Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"10302526-c446-42d2-8a70-97de89b14f7a is now active with session: 8908ee2e-89a1-4472-9ba8-f47ce8a5db8d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405557826Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405572807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286,"session_id":"8908ee2e-89a1-4472-9ba8-f47ce8a5db8d"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:31:49.405588317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 10 len 20480 data: 3 3 3 3 3 02/30 Write block 333 len 20480 data: 2 3 3 3 3 03/30 Write block 377 len 24576 data: 5 5 5 5 5 5 04/30 Write block 511 len 4096 data: 4 05/30 Flush 06/30 Write block 374 len 20480 data: 5 3 4 6 6 07/30 Flush 08/30 Write block 186 len 12288 data: 2 2 2 09/30 Read block 56 len 16384 10/30 Read block 219 len 24576 11/30 Write block 354 len 24576 data: 4 4 4 3 3 4 12/30 Read block 4 len 20480 13/30 Flush 14/30 Write block 184 len 24576 data: 2 2 3 3 3 2 15/30 Write block 147 len 12288 data: 5 5 4 16/30 Read block 111 len 24576 17/30 Write block 17 len 20480 data: 3 4 4 4 5 18/30 Read block 162 len 4096 19/30 Read block 515 len 32768 20/30 Flush 21/30 Flush 22/30 Write block 334 len 12288 data: 4 4 4 23/30 Write block 278 len 8192 data: 3 3 24/30 Write block 424 len 16384 data: 3 3 3 3 25/30 Write block 149 len 4096 data: 5 26/30 Read block 364 len 4096 27/30 Read block 415 len 4096 28/30 Write block 535 len 20480 data: 3 3 3 3 2 29/30 Read block 242 len 28672 30/30 Write block 274 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 3 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:31:49.432297136Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 54 54 54 F F F 1 020-039 26 26 26 54 54 54 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 24 24 24 48 48 48 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 22 22 22 43 43 43 F F F 7 140-159 26 26 26 54 54 54 T T T 8 160-179 19 19 19 38 38 38 F F F 9 180-199 26 26 26 54 54 54 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 21 21 21 41 41 41 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 24 24 24 49 49 49 T T T 14 280-299 23 23 23 46 46 46 F F F 15 300-319 22 22 22 44 44 44 F F F 16 320-339 26 26 26 51 51 51 T T T 17 340-359 26 26 26 53 53 53 F F F 18 360-379 26 26 26 52 52 52 F F F 19 380-399 26 26 26 51 51 51 F F F 20 400-419 24 24 24 48 48 48 F F F 21 420-439 25 25 25 50 50 50 T T T 22 440-459 25 25 25 50 50 50 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 22 22 22 43 43 43 T T T 27 540-559 23 23 23 46 46 46 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 26, Max flush: 54 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 54 54 54 F F F 1 020-039 26 26 26 54 54 54 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 24 24 24 48 48 48 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 22 22 22 43 43 43 F F F 7 140-159 27 27 27 55 55 55 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 26 26 26 54 54 54 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 21 21 21 41 41 41 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 27 27 27 55 55 55 F F F 14 280-299 23 23 23 46 46 46 F F F 15 300-319 22 22 22 44 44 44 F F F 16 320-339 27 27 27 55 55 55 F F F 17 340-359 26 26 26 53 53 53 F F F 18 360-379 26 26 26 52 52 52 F F F 19 380-399 26 26 26 51 51 51 F F F 20 400-419 24 24 24 48 48 48 F F F 21 420-439 27 27 27 55 55 55 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 27 27 27 55 55 55 F F F 27 540-559 23 23 23 46 46 46 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 16 16 16 31 31 31 F F F Max gen: 27, Max flush: 55 Begin loop 14 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.661875819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.662144233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.662166083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294} {"msg":"Crucible f734adf8-a522-440b-aa63-92c1e5439fe7 has session id: 492c190b-5dda-43b7-aefd-3315c60e99e0","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.662620079Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Upstairs opts: Upstairs UUID: f734adf8-a522-440b-aa63-92c1e5439fe7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.662662079Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Crucible stats registered with UUID: f734adf8-a522-440b-aa63-92c1e5439fe7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.662685779Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663047964Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294} {"msg":"f734adf8-a522-440b-aa63-92c1e5439fe7 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663126035Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663144585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663159705Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663182906Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663208286Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663234666Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663261067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663285057Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663311617Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663336817Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54002)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663451039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:57397)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.66348337Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:03.663702452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:03.663737003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:03.663755563Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663768723Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663782343Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.663883104Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.664212519Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.664239539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.66434915Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.664377451Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.664713655Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.664818676Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.665180371Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.665218392Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.665247052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.665286222Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.665315853Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:03.665332933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.66509107Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35487)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.665361183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.666254485Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.666288745Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.666769401Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667295508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667318438Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[0]R flush_numbers[0..12]: [54, 54, 25, 48, 48, 48, 43, 55, 38, 54, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667336628Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[0]R generation[0..12]: [26, 26, 14, 24, 24, 24, 22, 27, 19, 26, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667354579Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667372729Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[1]R flush_numbers[0..12]: [54, 54, 25, 48, 48, 48, 43, 55, 38, 54, 37, 41]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667396459Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[1]R generation[0..12]: [26, 26, 14, 24, 24, 24, 22, 27, 19, 26, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.66741426Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.66744184Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[2]R flush_numbers[0..12]: [54, 54, 25, 48, 48, 48, 43, 55, 38, 54, 37, 41]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:32:13.66747099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[2]R generation[0..12]: [26, 26, 14, 24, 24, 24, 22, 27, 19, 26, 18, 21]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667501061Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667533671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667563731Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667595092Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667615412Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667635152Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"":"downstairs","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667663753Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667692413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"0","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667720283Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"1","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667738033Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"client":"2","":"downstairs","session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667769494Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667798404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"f734adf8-a522-440b-aa63-92c1e5439fe7 is now active with session: 492c190b-5dda-43b7-aefd-3315c60e99e0","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667828975Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667847675Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294,"session_id":"492c190b-5dda-43b7-aefd-3315c60e99e0"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:13.667870645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 348 len 36864 data: 3 2 3 3 3 4 5 5 5 02/30 Read block 586 len 12288 03/30 Flush 04/30 Write block 274 len 12288 data: 3 2 2 05/30 Write block 586 len 28672 data: 2 2 2 2 4 5 4 06/30 Flush 07/30 Write block 360 len 8192 data: 3 4 08/30 Write block 234 len 12288 data: 5 5 4 09/30 Read block 530 len 12288 10/30 Write block 588 len 32768 data: 3 3 5 6 5 3 3 3 11/30 Write block 526 len 28672 data: 4 4 4 3 3 3 2 12/30 Write block 173 len 12288 data: 2 2 2 13/30 Write block 526 len 24576 data: 5 5 5 4 4 4 14/30 Read block 298 len 12288 15/30 Write block 43 len 4096 data: 3 16/30 Read block 310 len 16384 17/30 Write block 50 len 28672 data: 3 4 5 5 4 3 4 18/30 Write block 172 len 12288 data: 2 3 3 19/30 Read block 244 len 4096 20/30 Flush 21/30 Write block 138 len 20480 data: 3 3 2 2 3 22/30 Read block 62 len 8192 23/30 Write block 5 len 8192 data: 3 2 24/30 Write block 0 len 32768 data: 3 4 5 6 5 4 3 2 25/30 Write block 167 len 36864 data: 2 2 2 2 2 3 4 4 3 26/30 Write block 47 len 24576 data: 2 3 3 4 5 6 27/30 Write block 13 len 4096 data: 4 28/30 Read block 330 len 24576 29/30 Write block 544 len 36864 data: 4 3 3 3 4 7 8 7 6 30/30 Read block 521 len 36864 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 9 Done Done Done false 30 Acked 1029 Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 1 2 Done 9 10 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:13.694362091Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 54 54 54 T T T 1 020-039 26 26 26 54 54 54 F F F 2 040-059 28 28 28 58 58 58 T T T 3 060-079 24 24 24 48 48 48 F F F 4 080-099 24 24 24 48 48 48 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 22 22 22 43 43 43 T T T 7 140-159 27 27 27 55 55 55 T T T 8 160-179 28 28 28 58 58 58 T T T 9 180-199 26 26 26 54 54 54 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 28 28 28 58 58 58 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 23 23 23 46 46 46 F F F 15 300-319 22 22 22 44 44 44 F F F 16 320-339 27 27 27 55 55 55 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 26 26 26 51 51 51 F F F 20 400-419 24 24 24 48 48 48 F F F 21 420-439 27 27 27 55 55 55 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 23 23 23 46 46 46 T T T 28 560-579 8 8 8 15 15 15 F F F 29 580-599 28 28 28 58 58 58 F F F Max gen: 28, Max flush: 58 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 29 29 29 59 59 59 F F F 1 020-039 26 26 26 54 54 54 F F F 2 040-059 29 29 29 59 59 59 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 24 24 24 48 48 48 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 29 29 29 59 59 59 F F F 7 140-159 29 29 29 59 59 59 F F F 8 160-179 29 29 29 59 59 59 F F F 9 180-199 26 26 26 54 54 54 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 28 28 28 58 58 58 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 23 23 23 46 46 46 F F F 15 300-319 22 22 22 44 44 44 F F F 16 320-339 27 27 27 55 55 55 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 26 26 26 51 51 51 F F F 20 400-419 24 24 24 48 48 48 F F F 21 420-439 27 27 27 55 55 55 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 59 59 59 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 28 28 28 58 58 58 F F F Max gen: 29, Max flush: 59 Begin loop 15 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.935924517Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.936318622Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.936359102Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302} {"msg":"Crucible 4310b380-b81a-41e1-bb70-b51d57f05a2b has session id: 30bfdeeb-92ae-43bf-852e-085a3a2980ad","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.936692296Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Upstairs opts: Upstairs UUID: 4310b380-b81a-41e1-bb70-b51d57f05a2b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.936737747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Crucible stats registered with UUID: 4310b380-b81a-41e1-bb70-b51d57f05a2b","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.936761037Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937240393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302} {"msg":"4310b380-b81a-41e1-bb70-b51d57f05a2b active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937308074Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937336605Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937363235Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937388395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937411866Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937433676Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937458066Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937479596Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937504057Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.937527377Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:49275)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.9377389Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:59776)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.93777529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:27.938030183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:27.938067994Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:27.938136915Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.938154265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.938167865Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.938315207Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.938773163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.938799803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.938814713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.938828213Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.93933778Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.939443861Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.939840106Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.939878437Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.939907257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.939947018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.939976298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:27.939993568Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.939871457Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:39067)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.940037009Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.940983561Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.941023481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.94167048Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942284117Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942313198Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[0]R flush_numbers[0..12]: [59, 54, 59, 48, 48, 48, 59, 59, 59, 54, 37, 58]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942331878Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[0]R generation[0..12]: [29, 26, 29, 24, 24, 24, 29, 29, 29, 26, 18, 28]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942349828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942373058Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[1]R flush_numbers[0..12]: [59, 54, 59, 48, 48, 48, 59, 59, 59, 54, 37, 58]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942390929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[1]R generation[0..12]: [29, 26, 29, 24, 24, 24, 29, 29, 29, 26, 18, 28]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942408239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942425989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[2]R flush_numbers[0..12]: [59, 54, 59, 48, 48, 48, 59, 59, 59, 54, 37, 58]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.94244333Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[2]R generation[0..12]: [29, 26, 29, 24, 24, 24, 29, 29, 29, 26, 18, 28]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.94246469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.9424826Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.94249929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.94251838Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942546221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942575531Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"":"downstairs","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942595391Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942618992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"0","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942649492Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"1","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942683682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"client":"2","":"downstairs","session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942713383Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942744563Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"4310b380-b81a-41e1-bb70-b51d57f05a2b is now active with session: 30bfdeeb-92ae-43bf-852e-085a3a2980ad","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942773634Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942795464Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302,"session_id":"30bfdeeb-92ae-43bf-852e-085a3a2980ad"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:37.942811654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 16384 data: 6 7 6 5 02/30 Read block 516 len 12288 03/30 Flush 04/30 Flush 05/30 Write block 167 len 12288 data: 3 3 3 06/30 Read block 456 len 36864 07/30 Read block 232 len 28672 08/30 Write block 285 len 12288 data: 3 3 3 09/30 Read block 408 len 32768 10/30 Read block 327 len 28672 11/30 Read block 452 len 28672 12/30 Write block 237 len 4096 data: 3 13/30 Read block 22 len 12288 14/30 Read block 468 len 20480 15/30 Read block 452 len 16384 16/30 Read block 306 len 24576 17/30 Write block 129 len 8192 data: 2 2 18/30 Write block 582 len 8192 data: 2 2 19/30 Read block 337 len 8192 20/30 Flush 21/30 Write block 174 len 40960 data: 5 4 2 2 2 2 2 2 2 2 22/30 Write block 177 len 16384 data: 3 3 3 3 23/30 Read block 109 len 8192 24/30 Flush 25/30 Flush 26/30 Write block 87 len 16384 data: 2 3 4 4 27/30 Read block 409 len 28672 28/30 Write block 9 len 40960 data: 3 4 4 4 5 4 2 3 4 5 29/30 Read block 41 len 28672 30/30 Write block 129 len 28672 data: 3 3 2 2 2 2 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:37.970175421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 60 60 60 T T T 1 020-039 26 26 26 54 54 54 F F F 2 040-059 29 29 29 59 59 59 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 24 24 24 48 48 48 T T T 5 100-119 24 24 24 48 48 48 F F F 6 120-139 30 30 30 62 62 62 T T T 7 140-159 29 29 29 59 59 59 F F F 8 160-179 30 30 30 63 63 63 F F F 9 180-199 30 30 30 63 63 63 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 30 30 30 62 62 62 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 22 22 22 44 44 44 F F F 16 320-339 27 27 27 55 55 55 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 26 26 26 51 51 51 F F F 20 400-419 24 24 24 48 48 48 F F F 21 420-439 27 27 27 55 55 55 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 59 59 59 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 30 30 30 62 62 62 F F F Max gen: 30, Max flush: 63 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 64 64 64 F F F 1 020-039 26 26 26 54 54 54 F F F 2 040-059 29 29 29 59 59 59 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 31 31 31 64 64 64 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 31 31 31 64 64 64 F F F 7 140-159 29 29 29 59 59 59 F F F 8 160-179 30 30 30 63 63 63 F F F 9 180-199 30 30 30 63 63 63 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 30 30 30 62 62 62 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 22 22 22 44 44 44 F F F 16 320-339 27 27 27 55 55 55 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 26 26 26 51 51 51 F F F 20 400-419 24 24 24 48 48 48 F F F 21 420-439 27 27 27 55 55 55 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 59 59 59 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 30 30 30 62 62 62 F F F Max gen: 31, Max flush: 64 Begin loop 16 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.173624353Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174028338Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174062499Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310} {"msg":"Crucible 1817f7cd-d178-4a9b-bc2e-d65d3958c450 has session id: 02c0706b-e767-4a22-a9fc-b0b10e05bae4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174382443Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Upstairs opts: Upstairs UUID: 1817f7cd-d178-4a9b-bc2e-d65d3958c450, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174451864Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Crucible stats registered with UUID: 1817f7cd-d178-4a9b-bc2e-d65d3958c450","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174485764Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174812128Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310} {"msg":"1817f7cd-d178-4a9b-bc2e-d65d3958c450 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174852049Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174867039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174881779Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174895559Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.174909369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.17492279Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.1749367Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.175004141Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.175023001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.175037091Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43956)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.175253724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_connection connected from Ok(127.0.0.1:39530)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.175295904Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:52.175475967Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:52.175505677Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:32:52.175586808Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.175604028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.175617758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.17576611Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.176174465Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.176239176Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.176301107Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.176320137Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.176685742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.176798743Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.177216629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.177244539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.177259129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.17728393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.17729877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:32:52.17731238Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.176797333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42164)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.177087107Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.17812792Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.17815473Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.178685557Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179213374Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179231824Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[0]R flush_numbers[0..12]: [64, 54, 59, 48, 64, 48, 64, 59, 63, 63, 37, 62]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179246154Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[0]R generation[0..12]: [31, 26, 29, 24, 31, 24, 31, 29, 30, 30, 18, 30]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179260335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179274905Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[1]R flush_numbers[0..12]: [64, 54, 59, 48, 64, 48, 64, 59, 63, 63, 37, 62]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179293215Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[1]R generation[0..12]: [31, 26, 29, 24, 31, 24, 31, 29, 30, 30, 18, 30]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179307395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179321875Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[2]R flush_numbers[0..12]: [64, 54, 59, 48, 64, 48, 64, 59, 63, 63, 37, 62]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179335805Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[2]R generation[0..12]: [31, 26, 29, 24, 31, 24, 31, 29, 30, 30, 18, 30]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179349546Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179363946Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Max found gen is 32","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-07-10T15:33:02.179384306Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179401126Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179417537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179434037Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"":"downstairs","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179450567Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179465747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"0","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179482387Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"1","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179498718Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"client":"2","":"downstairs","session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179515428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179533548Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"1817f7cd-d178-4a9b-bc2e-d65d3958c450 is now active with session: 02c0706b-e767-4a22-a9fc-b0b10e05bae4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179549768Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179564888Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310,"session_id":"02c0706b-e767-4a22-a9fc-b0b10e05bae4"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:02.179580339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 192 len 28672 data: 5 4 5 5 4 5 3 02/30 Flush 03/30 Write block 433 len 8192 data: 2 2 04/30 Write block 310 len 8192 data: 2 2 05/30 Flush 06/30 Write block 156 len 16384 data: 5 4 6 6 07/30 Flush 08/30 Write block 135 len 20480 data: 4 3 3 4 4 09/30 Read block 385 len 20480 10/30 Read block 425 len 28672 11/30 Read block 307 len 8192 12/30 Read block 343 len 8192 13/30 Read block 560 len 36864 14/30 Read block 434 len 20480 15/30 Write block 158 len 40960 data: 7 7 4 3 3 3 3 3 3 4 16/30 Read block 196 len 36864 17/30 Flush 18/30 Write block 186 len 28672 data: 4 4 4 3 3 5 6 19/30 Write block 174 len 24576 data: 6 5 3 4 4 4 20/30 Read block 84 len 40960 21/30 Read block 592 len 32768 22/30 Read block 126 len 8192 23/30 Read block 178 len 16384 24/30 Flush 25/30 Write block 24 len 8192 data: 4 4 26/30 Write block 388 len 24576 data: 2 2 2 2 2 2 27/30 Read block 83 len 24576 28/30 Read block 166 len 24576 29/30 Write block 427 len 20480 data: 4 3 3 3 3 30/30 Write block 336 len 40960 data: 5 4 3 5 6 6 6 5 4 3 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 5 Done Done Done false 31 Acked 1030 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:33:02.209254055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 64 64 64 F F F 1 020-039 26 26 26 54 54 54 T T T 2 040-059 29 29 29 59 59 59 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 31 31 31 64 64 64 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 32 32 32 68 68 68 F F F 7 140-159 32 32 32 68 68 68 F F F 8 160-179 32 32 32 70 70 70 F F F 9 180-199 32 32 32 70 70 70 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 30 30 30 62 62 62 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 27 27 27 55 55 55 T T T 17 340-359 28 28 28 56 56 56 T T T 18 360-379 28 28 28 58 58 58 F F F 19 380-399 26 26 26 51 51 51 T T T 20 400-419 24 24 24 48 48 48 F F F 21 420-439 32 32 32 66 66 66 T T T 22 440-459 25 25 25 50 50 50 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 59 59 59 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 30 30 30 62 62 62 F F F Max gen: 32, Max flush: 70 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 64 64 64 F F F 1 020-039 33 33 33 71 71 71 F F F 2 040-059 29 29 29 59 59 59 F F F 3 060-079 24 24 24 48 48 48 F F F 4 080-099 31 31 31 64 64 64 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 32 32 32 68 68 68 F F F 7 140-159 32 32 32 68 68 68 F F F 8 160-179 32 32 32 70 70 70 F F F 9 180-199 32 32 32 70 70 70 F F F 10 200-219 18 18 18 37 37 37 F F F 11 220-239 30 30 30 62 62 62 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 33 33 33 71 71 71 F F F 17 340-359 33 33 33 71 71 71 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 33 33 33 71 71 71 F F F 20 400-419 24 24 24 48 48 48 F F F 21 420-439 33 33 33 71 71 71 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 59 59 59 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 30 30 30 62 62 62 F F F Max gen: 33, Max flush: 71 Begin loop 17 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.434319521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.434825228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.434855008Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318} {"msg":"Crucible 47b26bdf-18b1-472c-8219-fc52284d3cc7 has session id: d274f077-30ed-471f-81c4-6ccd3c72bb54","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435151602Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Upstairs opts: Upstairs UUID: 47b26bdf-18b1-472c-8219-fc52284d3cc7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435200533Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Crucible stats registered with UUID: 47b26bdf-18b1-472c-8219-fc52284d3cc7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435224673Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435479636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318} {"msg":"47b26bdf-18b1-472c-8219-fc52284d3cc7 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435504587Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435519767Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435534457Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435556447Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435581877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435606238Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435629748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435651969Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435676239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.435700719Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:33:16.435962682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:33:16.436010693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_connection connected from Ok(127.0.0.1:48526)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.436038873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:33:16.436056554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.436069404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.436082774Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_connection connected from Ok(127.0.0.1:35882)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.436104304Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.436129175Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.436689602Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.436716652Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.436731342Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.436744952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.437275789Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.437387131Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.437782425Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.437817366Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.437844446Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.437864666Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.437886537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:16.437908127Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.43736929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_connection connected from Ok(127.0.0.1:45123)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.437574602Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.438649746Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.438679807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439213103Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.4397439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.43976644Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[0]R flush_numbers[0..12]: [64, 71, 59, 48, 64, 48, 68, 68, 70, 70, 37, 62]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439781621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[0]R generation[0..12]: [31, 33, 29, 24, 31, 24, 32, 32, 32, 32, 18, 30]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439796151Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439810481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[1]R flush_numbers[0..12]: [64, 71, 59, 48, 64, 48, 68, 68, 70, 70, 37, 62]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439824421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[1]R generation[0..12]: [31, 33, 29, 24, 31, 24, 32, 32, 32, 32, 18, 30]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439842651Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439857402Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[2]R flush_numbers[0..12]: [64, 71, 59, 48, 64, 48, 68, 68, 70, 70, 37, 62]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439871292Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[2]R generation[0..12]: [31, 33, 29, 24, 31, 24, 32, 32, 32, 32, 18, 30]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439884922Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:33:26.439898942Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439915992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439932572Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439948683Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439965083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"":"downstairs","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439981653Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.439996503Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"0","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.440013233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"1","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.440029674Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"client":"2","":"downstairs","session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.440046224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.440061484Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"47b26bdf-18b1-472c-8219-fc52284d3cc7 is now active with session: d274f077-30ed-471f-81c4-6ccd3c72bb54","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.440080794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.440096065Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318,"session_id":"d274f077-30ed-471f-81c4-6ccd3c72bb54"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:26.440111365Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 567 len 8192 data: 2 2 02/30 Read block 576 len 32768 03/30 Write block 433 len 32768 data: 3 3 2 3 3 2 4 4 04/30 Write block 532 len 36864 data: 3 3 3 4 4 4 4 3 2 05/30 Read block 67 len 32768 06/30 Read block 462 len 32768 07/30 Write block 436 len 4096 data: 4 08/30 Read block 306 len 12288 09/30 Read block 66 len 12288 10/30 Write block 39 len 32768 data: 2 3 3 4 4 3 3 3 11/30 Read block 223 len 24576 12/30 Write block 229 len 32768 data: 6 4 3 3 5 6 6 5 13/30 Write block 145 len 24576 data: 4 4 6 6 6 4 14/30 Read block 454 len 16384 15/30 Read block 283 len 40960 16/30 Flush 17/30 Flush 18/30 Write block 573 len 4096 data: 2 19/30 Write block 413 len 8192 data: 4 3 20/30 Write block 52 len 40960 data: 7 6 5 4 5 3 3 2 2 3 21/30 Read block 70 len 28672 22/30 Read block 122 len 40960 23/30 Read block 414 len 24576 24/30 Read block 192 len 32768 25/30 Write block 441 len 4096 data: 5 26/30 Read block 109 len 32768 27/30 Read block 232 len 20480 28/30 Write block 550 len 16384 data: 9 8 7 5 29/30 Write block 576 len 20480 data: 2 2 2 2 2 30/30 Write block 197 len 40960 data: 6 4 2 2 2 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Write 10 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Write 5 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:33:26.466286617Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 64 64 64 F F F 1 020-039 34 34 34 72 72 72 F F F 2 040-059 34 34 34 72 72 72 T T T 3 060-079 24 24 24 48 48 48 T T T 4 080-099 31 31 31 64 64 64 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 32 32 32 68 68 68 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 32 32 32 70 70 70 F F F 9 180-199 32 32 32 70 70 70 T T T 10 200-219 18 18 18 37 37 37 T T T 11 220-239 34 34 34 72 72 72 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 33 33 33 71 71 71 F F F 17 340-359 33 33 33 71 71 71 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 33 33 33 71 71 71 F F F 20 400-419 24 24 24 48 48 48 T T T 21 420-439 34 34 34 72 72 72 F F F 22 440-459 34 34 34 72 72 72 T T T 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 34 34 34 72 72 72 F F F 27 540-559 34 34 34 72 72 72 T T T 28 560-579 34 34 34 72 72 72 T T T 29 580-599 30 30 30 62 62 62 T T T Max gen: 34, Max flush: 72 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 64 64 64 F F F 1 020-039 34 34 34 72 72 72 F F F 2 040-059 35 35 35 73 73 73 F F F 3 060-079 35 35 35 73 73 73 F F F 4 080-099 31 31 31 64 64 64 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 32 32 32 68 68 68 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 32 32 32 70 70 70 F F F 9 180-199 35 35 35 73 73 73 F F F 10 200-219 35 35 35 73 73 73 F F F 11 220-239 34 34 34 72 72 72 F F F 12 240-259 24 24 24 49 49 49 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 33 33 33 71 71 71 F F F 17 340-359 33 33 33 71 71 71 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 33 33 33 71 71 71 F F F 20 400-419 35 35 35 73 73 73 F F F 21 420-439 34 34 34 72 72 72 F F F 22 440-459 35 35 35 73 73 73 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 34 34 34 72 72 72 F F F 27 540-559 35 35 35 73 73 73 F F F 28 560-579 35 35 35 73 73 73 F F F 29 580-599 35 35 35 73 73 73 F F F Max gen: 35, Max flush: 73 Begin loop 18 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.748164904Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.74862165Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.74865153Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326} {"msg":"Crucible 435cd84f-a372-46cf-9b86-db9e1270ac30 has session id: a768abcc-490e-4962-baba-84033f293e18","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.748979005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Upstairs opts: Upstairs UUID: 435cd84f-a372-46cf-9b86-db9e1270ac30, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749053646Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Crucible stats registered with UUID: 435cd84f-a372-46cf-9b86-db9e1270ac30","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749079566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749348239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326} {"msg":"435cd84f-a372-46cf-9b86-db9e1270ac30 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749446231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749464331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749479371Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749493301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749507221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749520461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749533892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749610342Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749629563Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749643723Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_connection connected from Ok(127.0.0.1:59777)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.749930257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:33:40.750077099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_connection connected from Ok(127.0.0.1:64401)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.750118749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:33:40.750149659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:33:40.75017874Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.75020108Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.75022598Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.750372372Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.750812498Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.750838608Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.750853198Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.750871338Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.751421976Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.751532027Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.751923072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.751954522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.751969923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.751989193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.752003033Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:40.752016653Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.751520327Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_connection connected from Ok(127.0.0.1:47771)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.75178779Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.753009236Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.753038136Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.753597753Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.75412525Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.75414899Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[0]R flush_numbers[0..12]: [64, 72, 73, 73, 64, 48, 68, 72, 70, 73, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.75416422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[0]R generation[0..12]: [31, 34, 35, 35, 31, 24, 32, 34, 32, 35, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754178891Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754193571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[1]R flush_numbers[0..12]: [64, 72, 73, 73, 64, 48, 68, 72, 70, 73, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754213281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[1]R generation[0..12]: [31, 34, 35, 35, 31, 24, 32, 34, 32, 35, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754227641Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754241861Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[2]R flush_numbers[0..12]: [64, 72, 73, 73, 64, 48, 68, 72, 70, 73, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754255621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[2]R generation[0..12]: [31, 34, 35, 35, 31, 24, 32, 34, 32, 35, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754269081Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754283292Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754296442Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754312902Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754326182Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754339473Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"":"downstairs","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754352693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754364803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"0","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754378113Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"1","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754391363Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"client":"2","":"downstairs","session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754405043Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754419534Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"435cd84f-a372-46cf-9b86-db9e1270ac30 is now active with session: a768abcc-490e-4962-baba-84033f293e18","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754432644Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754444714Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326,"session_id":"a768abcc-490e-4962-baba-84033f293e18"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:33:50.754456934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 553 len 8192 data: 6 5 02/30 Read block 254 len 4096 03/30 Write block 351 len 20480 data: 4 4 5 6 6 04/30 Read block 582 len 16384 05/30 Read block 376 len 4096 06/30 Read block 544 len 40960 07/30 Write block 168 len 4096 data: 4 08/30 Read block 349 len 28672 09/30 Write block 580 len 12288 data: 3 2 3 10/30 Read block 4 len 4096 11/30 Write block 182 len 4096 data: 3 12/30 Read block 402 len 20480 13/30 Read block 98 len 24576 14/30 Write block 134 len 24576 data: 3 5 4 4 5 5 15/30 Flush 16/30 Write block 18 len 16384 data: 6 5 5 6 17/30 Read block 539 len 12288 18/30 Read block 55 len 16384 19/30 Read block 310 len 40960 20/30 Flush 21/30 Write block 83 len 20480 data: 2 2 2 2 3 22/30 Read block 503 len 12288 23/30 Write block 544 len 4096 data: 5 24/30 Read block 232 len 20480 25/30 Write block 243 len 32768 data: 2 3 3 4 5 5 5 5 26/30 Flush 27/30 Write block 337 len 4096 data: 5 28/30 Write block 131 len 16384 data: 3 3 3 4 29/30 Write block 127 len 24576 data: 2 2 4 4 4 4 30/30 Write block 325 len 24576 data: 5 4 4 4 4 4 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Write 6 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:33:50.780509285Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 75 75 75 F F F 1 020-039 36 36 36 75 75 75 F F F 2 040-059 35 35 35 73 73 73 F F F 3 060-079 35 35 35 73 73 73 F F F 4 080-099 36 36 36 76 76 76 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 36 36 36 74 74 74 T T T 7 140-159 34 34 34 72 72 72 F F F 8 160-179 36 36 36 74 74 74 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 35 35 35 73 73 73 F F F 11 220-239 34 34 34 72 72 72 F F F 12 240-259 36 36 36 76 76 76 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 33 33 33 71 71 71 T T T 17 340-359 36 36 36 74 74 74 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 33 33 33 71 71 71 F F F 20 400-419 35 35 35 73 73 73 F F F 21 420-439 34 34 34 72 72 72 F F F 22 440-459 35 35 35 73 73 73 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 34 34 34 72 72 72 F F F 27 540-559 36 36 36 76 76 76 F F F 28 560-579 35 35 35 73 73 73 F F F 29 580-599 36 36 36 74 74 74 F F F Max gen: 36, Max flush: 76 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 75 75 75 F F F 1 020-039 36 36 36 75 75 75 F F F 2 040-059 35 35 35 73 73 73 F F F 3 060-079 35 35 35 73 73 73 F F F 4 080-099 36 36 36 76 76 76 F F F 5 100-119 24 24 24 48 48 48 F F F 6 120-139 37 37 37 77 77 77 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 36 36 36 74 74 74 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 35 35 35 73 73 73 F F F 11 220-239 34 34 34 72 72 72 F F F 12 240-259 36 36 36 76 76 76 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 37 37 37 77 77 77 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 33 33 33 71 71 71 F F F 20 400-419 35 35 35 73 73 73 F F F 21 420-439 34 34 34 72 72 72 F F F 22 440-459 35 35 35 73 73 73 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 34 34 34 72 72 72 F F F 27 540-559 36 36 36 76 76 76 F F F 28 560-579 35 35 35 73 73 73 F F F 29 580-599 36 36 36 74 74 74 F F F Max gen: 37, Max flush: 77 Begin loop 19 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.969706945Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.969970578Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.969988779Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334} {"msg":"Crucible c226f901-cedb-450f-a877-619e2409a663 has session id: d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.970375224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Upstairs opts: Upstairs UUID: c226f901-cedb-450f-a877-619e2409a663, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.970421924Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Crucible stats registered with UUID: c226f901-cedb-450f-a877-619e2409a663","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.970438884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.970785129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334} {"msg":"c226f901-cedb-450f-a877-619e2409a663 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.970815119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.97083964Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.97086542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.97089825Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.970959521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.970988201Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.971037672Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.971070643Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.971175264Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.971346026Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_connection connected from Ok(127.0.0.1:41570)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.971378656Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:04.971488048Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:04.971532188Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_connection connected from Ok(127.0.0.1:49234)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.971562319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:04.971593079Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.971618479Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.97163996Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.97166217Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.972056525Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.972083055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.972098396Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.972116986Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.972672733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.972800024Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.973188859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.9732164Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.97323539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.973305561Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.973325501Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:04.973339451Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.973036257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_connection connected from Ok(127.0.0.1:37053)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.97322408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.974145482Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.974173622Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.974732529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975265296Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975288056Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[0]R flush_numbers[0..12]: [75, 75, 73, 73, 76, 48, 77, 72, 74, 74, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975306326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[0]R generation[0..12]: [36, 36, 35, 35, 36, 24, 37, 34, 36, 36, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975323917Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975341687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[1]R flush_numbers[0..12]: [75, 75, 73, 73, 76, 48, 77, 72, 74, 74, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975359657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[1]R generation[0..12]: [36, 36, 35, 35, 36, 24, 37, 34, 36, 36, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975382837Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975400987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[2]R flush_numbers[0..12]: [75, 75, 73, 73, 76, 48, 77, 72, 74, 74, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975418298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[2]R generation[0..12]: [36, 36, 35, 35, 36, 24, 37, 34, 36, 36, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975438888Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975456698Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975473068Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975489689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975505759Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975522139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"":"downstairs","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975538909Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975554049Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"0","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.97557076Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"1","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.97558802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"client":"2","":"downstairs","session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.97560508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.97562047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"c226f901-cedb-450f-a877-619e2409a663 is now active with session: d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-07-10T15:34:14.975639311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975657911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334,"session_id":"d9c6a10c-ec50-4d6c-ad25-97cf8d6ea024"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:14.975673521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 73 len 28672 data: 3 2 2 2 2 2 2 03/30 Read block 246 len 32768 04/30 Read block 407 len 28672 05/30 Flush 06/30 Write block 137 len 12288 data: 5 6 6 07/30 Read block 261 len 40960 08/30 Read block 427 len 36864 09/30 Write block 120 len 20480 data: 3 3 2 2 2 10/30 Write block 108 len 20480 data: 2 2 2 2 2 11/30 Write block 254 len 20480 data: 5 7 4 4 4 12/30 Read block 585 len 8192 13/30 Write block 446 len 32768 data: 4 5 5 3 4 4 4 4 14/30 Read block 309 len 28672 15/30 Write block 423 len 40960 data: 4 4 4 4 5 4 4 4 4 3 16/30 Read block 478 len 28672 17/30 Read block 574 len 12288 18/30 Write block 435 len 28672 data: 3 5 4 3 5 5 6 19/30 Read block 9 len 40960 20/30 Write block 536 len 36864 data: 5 5 5 4 3 2 3 4 6 21/30 Read block 277 len 36864 22/30 Write block 422 len 32768 data: 4 5 5 5 5 6 5 5 23/30 Read block 68 len 36864 24/30 Read block 503 len 32768 25/30 Read block 104 len 16384 26/30 Read block 355 len 12288 27/30 Read block 208 len 4096 28/30 Write block 441 len 12288 data: 7 5 5 29/30 Read block 350 len 40960 30/30 Write block 66 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 7 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 3 Done Sent Done false 30 Acked 1029 Read 10 Sent Sent Done false 31 Acked 1030 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 2 3 1 6 Done 11 10 12 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:15.0031547Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 75 75 75 F F F 1 020-039 36 36 36 75 75 75 F F F 2 040-059 35 35 35 73 73 73 F F F 3 060-079 38 38 38 79 79 79 T T T 4 080-099 36 36 36 76 76 76 F F F 5 100-119 38 38 38 80 80 80 F F F 6 120-139 38 38 38 80 80 80 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 36 36 36 74 74 74 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 35 35 35 73 73 73 F F F 11 220-239 34 34 34 72 72 72 F F F 12 240-259 38 38 38 80 80 80 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 37 37 37 77 77 77 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 33 33 33 71 71 71 F F F 20 400-419 35 35 35 73 73 73 F F F 21 420-439 38 38 38 80 80 80 T T T 22 440-459 38 38 38 80 80 80 T T T 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 34 34 34 72 72 72 T T T 27 540-559 36 36 36 76 76 76 T T T 28 560-579 35 35 35 73 73 73 F F F 29 580-599 36 36 36 74 74 74 F F F Max gen: 38, Max flush: 80 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 75 75 75 F F F 1 020-039 36 36 36 75 75 75 F F F 2 040-059 35 35 35 73 73 73 F F F 3 060-079 39 39 39 81 81 81 F F F 4 080-099 36 36 36 76 76 76 F F F 5 100-119 38 38 38 80 80 80 F F F 6 120-139 38 38 38 80 80 80 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 36 36 36 74 74 74 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 35 35 35 73 73 73 F F F 11 220-239 34 34 34 72 72 72 F F F 12 240-259 38 38 38 80 80 80 F F F 13 260-279 28 28 28 57 57 57 F F F 14 280-299 30 30 30 62 62 62 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 37 37 37 77 77 77 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 28 28 28 58 58 58 F F F 19 380-399 33 33 33 71 71 71 F F F 20 400-419 35 35 35 73 73 73 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 39 39 39 81 81 81 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 39 39 39 81 81 81 F F F 27 540-559 39 39 39 81 81 81 F F F 28 560-579 35 35 35 73 73 73 F F F 29 580-599 36 36 36 74 74 74 F F F Max gen: 39, Max flush: 81 Begin loop 20 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.233850467Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.234558236Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.234587136Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342} {"msg":"Crucible a061d282-4cb9-4682-a85e-45b81f0f0634 has session id: 78abd078-4cbe-4287-a93e-2b9e106c0122","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.234629917Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Upstairs opts: Upstairs UUID: a061d282-4cb9-4682-a85e-45b81f0f0634, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.234669248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Crucible stats registered with UUID: a061d282-4cb9-4682-a85e-45b81f0f0634","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.234693528Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235112833Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342} {"msg":"a061d282-4cb9-4682-a85e-45b81f0f0634 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235147504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235162024Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235176254Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235190184Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235203814Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235217104Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235236765Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235285225Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235303636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235317666Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_connection connected from Ok(127.0.0.1:36372)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235532459Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_connection connected from Ok(127.0.0.1:59316)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235575539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:29.235746161Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:29.235775492Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:29.235792742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235805642Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235819172Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.235934724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.23647576Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.236502601Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.236517281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.236531021Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.237104348Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.23722047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.237581705Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.237618395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.237638945Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.237740687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.237758597Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:29.237772067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.237146989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_connection connected from Ok(127.0.0.1:54188)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.237508414Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.238496166Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.238523977Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239104594Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239697032Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239716392Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[0]R flush_numbers[0..12]: [75, 75, 73, 81, 76, 80, 80, 72, 74, 74, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239743912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[0]R generation[0..12]: [36, 36, 35, 39, 36, 38, 38, 34, 36, 36, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239759222Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239773882Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[1]R flush_numbers[0..12]: [75, 75, 73, 81, 76, 80, 80, 72, 74, 74, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239792973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[1]R generation[0..12]: [36, 36, 35, 39, 36, 38, 38, 34, 36, 36, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239807193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239821603Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[2]R flush_numbers[0..12]: [75, 75, 73, 81, 76, 80, 80, 72, 74, 74, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239835413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[2]R generation[0..12]: [36, 36, 35, 39, 36, 38, 38, 34, 36, 36, 35, 34]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239849063Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239863304Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239876344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:34:39.239889824Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239903154Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239916194Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"":"downstairs","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239929464Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239941485Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"0","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239954785Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"1","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239967965Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"client":"2","":"downstairs","session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.239981195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.240009396Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"a061d282-4cb9-4682-a85e-45b81f0f0634 is now active with session: 78abd078-4cbe-4287-a93e-2b9e106c0122","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.240028166Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.240046126Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342,"session_id":"78abd078-4cbe-4287-a93e-2b9e106c0122"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:39.240059106Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 4096 data: 3 02/30 Write block 280 len 40960 data: 3 3 3 3 2 4 4 4 3 3 03/30 Read block 349 len 36864 04/30 Write block 170 len 16384 data: 3 3 4 5 05/30 Read block 44 len 40960 06/30 Flush 07/30 Write block 39 len 12288 data: 3 4 4 08/30 Write block 370 len 20480 data: 5 4 4 4 6 09/30 Read block 206 len 32768 10/30 Read block 89 len 28672 11/30 Flush 12/30 Write block 227 len 20480 data: 6 6 7 5 4 13/30 Read block 143 len 20480 14/30 Read block 64 len 20480 15/30 Write block 413 len 4096 data: 5 16/30 Write block 562 len 4096 data: 3 17/30 Flush 18/30 Write block 275 len 24576 data: 3 3 3 4 4 4 19/30 Read block 514 len 32768 20/30 Flush 21/30 Write block 578 len 8192 data: 3 3 22/30 Read block 521 len 8192 23/30 Read block 298 len 8192 24/30 Read block 341 len 8192 25/30 Flush 26/30 Flush 27/30 Write block 377 len 4096 data: 7 28/30 Read block 178 len 28672 29/30 Write block 578 len 12288 data: 4 4 4 30/30 Write block 525 len 28672 data: 2 6 6 6 5 5 5 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 3 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:39.269664382Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 75 75 75 F F F 1 020-039 40 40 40 83 83 83 F F F 2 040-059 40 40 40 83 83 83 F F F 3 060-079 39 39 39 81 81 81 F F F 4 080-099 36 36 36 76 76 76 F F F 5 100-119 38 38 38 80 80 80 F F F 6 120-139 38 38 38 80 80 80 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 35 35 35 73 73 73 F F F 11 220-239 40 40 40 84 84 84 F F F 12 240-259 38 38 38 80 80 80 F F F 13 260-279 40 40 40 86 86 86 F F F 14 280-299 40 40 40 86 86 86 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 37 37 37 77 77 77 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 40 40 40 83 83 83 T T T 19 380-399 33 33 33 71 71 71 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 39 39 39 81 81 81 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 39 39 39 81 81 81 T T T 27 540-559 39 39 39 81 81 81 F F F 28 560-579 40 40 40 87 87 87 T T T 29 580-599 36 36 36 74 74 74 T T T Max gen: 40, Max flush: 87 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 75 75 75 F F F 1 020-039 40 40 40 83 83 83 F F F 2 040-059 40 40 40 83 83 83 F F F 3 060-079 39 39 39 81 81 81 F F F 4 080-099 36 36 36 76 76 76 F F F 5 100-119 38 38 38 80 80 80 F F F 6 120-139 38 38 38 80 80 80 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 35 35 35 73 73 73 F F F 11 220-239 40 40 40 84 84 84 F F F 12 240-259 38 38 38 80 80 80 F F F 13 260-279 40 40 40 86 86 86 F F F 14 280-299 40 40 40 86 86 86 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 37 37 37 77 77 77 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 41 41 41 88 88 88 F F F 19 380-399 33 33 33 71 71 71 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 39 39 39 81 81 81 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 41 41 41 88 88 88 F F F 27 540-559 39 39 39 81 81 81 F F F 28 560-579 41 41 41 88 88 88 F F F 29 580-599 41 41 41 88 88 88 F F F Max gen: 41, Max flush: 88 Begin loop 21 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.484328926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.484690391Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.484708751Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350} {"msg":"Crucible a2b99871-a569-4769-acd4-c701d6a689a1 has session id: 086beef0-5202-413e-b0a9-7a8f73e75d6c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485207027Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Upstairs opts: Upstairs UUID: a2b99871-a569-4769-acd4-c701d6a689a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485239718Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Crucible stats registered with UUID: a2b99871-a569-4769-acd4-c701d6a689a1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485258548Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485646363Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350} {"msg":"a2b99871-a569-4769-acd4-c701d6a689a1 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485687823Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485704163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485718804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485732634Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485752484Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485776575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485800855Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485823315Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485846175Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.485870006Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42422)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.486006907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:53.486049668Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:32918)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.486116419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:53.486148809Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:34:53.48619262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.48621394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.48623524Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.486388632Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.486950939Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.48698574Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.487055521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.487093761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.487594318Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.487633908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.488203735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.488230466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.488245846Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.488263986Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.488278416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:34:53.488291946Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.487261923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37387)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.487474596Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.488701301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.488728952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489223778Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489768055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489786575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[0]R flush_numbers[0..12]: [75, 83, 83, 81, 76, 80, 80, 72, 82, 74, 73, 84]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489800995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[0]R generation[0..12]: [36, 40, 40, 39, 36, 38, 38, 34, 40, 36, 35, 40]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489815086Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489829626Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[1]R flush_numbers[0..12]: [75, 83, 83, 81, 76, 80, 80, 72, 82, 74, 73, 84]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489843626Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[1]R generation[0..12]: [36, 40, 40, 39, 36, 38, 38, 34, 40, 36, 35, 40]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489857526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489876556Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[2]R flush_numbers[0..12]: [75, 83, 83, 81, 76, 80, 80, 72, 82, 74, 73, 84]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489891027Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[2]R generation[0..12]: [36, 40, 40, 39, 36, 38, 38, 34, 40, 36, 35, 40]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489904867Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489919037Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489932107Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":Wait for a query_work_queue command to finish before sending IO "Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489949437Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489962537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489975628Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"":"downstairs","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.489988878Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.490001118Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"0","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.490014448Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"1","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.490027648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"client":"2","":"downstairs","session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.490041189Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.490053609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"a2b99871-a569-4769-acd4-c701d6a689a1 is now active with session: 086beef0-5202-413e-b0a9-7a8f73e75d6c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.490066619Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.490080809Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350,"session_id":"086beef0-5202-413e-b0a9-7a8f73e75d6c"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:03.490093269Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 130 len 4096 data: 5 02/30 Write block 29 len 4096 data: 4 03/30 Read block 188 len 24576 04/30 Read block 119 len 36864 05/30 Write block 536 len 40960 data: 6 6 6 5 4 3 4 5 7 4 06/30 Flush 07/30 Write block 59 len 28672 data: 3 3 4 3 3 4 4 08/30 Write block 251 len 40960 data: 6 5 5 6 8 5 5 5 6 6 09/30 Write block 248 len 28672 data: 6 6 6 7 6 6 7 10/30 Read block 156 len 36864 11/30 Flush 12/30 Flush 13/30 Write block 95 len 28672 data: 6 5 5 5 5 5 3 14/30 Write block 207 len 32768 data: 4 4 4 5 6 5 4 3 15/30 Write block 557 len 8192 data: 6 4 16/30 Read block 374 len 32768 17/30 Read block 494 len 24576 18/30 Read block 545 len 32768 19/30 Write block 441 len 20480 data: 8 6 6 5 4 20/30 Read block 589 len 24576 21/30 Read block 355 len 8192 22/30 Read block 294 len 12288 23/30 Read block 111 len 4096 24/30 Read block 52 len 32768 25/30 Read block 349 len 8192 26/30 Write block 234 len 12288 data: 7 7 6 27/30 Read block 262 len 8192 28/30 Read block 570 len 28672 29/30 Write block 108 len 36864 data: 3 3 3 3 3 2 2 3 3 30/30 Write block 373 len 28672 data: 5 7 4 5 8 7 6 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Read 6 Done Done Done false 19 Acked 1018 Read 8 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 6 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 9 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:35:03.518991026Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 75 75 75 F F F 1 020-039 42 42 42 89 89 89 F F F 2 040-059 42 42 42 90 90 90 F F F 3 060-079 42 42 42 90 90 90 F F F 4 080-099 42 42 42 92 92 92 F F F 5 100-119 42 42 42 92 92 92 T T T 6 120-139 42 42 42 89 89 89 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 42 42 42 92 92 92 F F F 11 220-239 40 40 40 84 84 84 T T T 12 240-259 42 42 42 90 90 90 F F F 13 260-279 42 42 42 90 90 90 F F F 14 280-299 40 40 40 86 86 86 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 37 37 37 77 77 77 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 41 41 41 88 88 88 T T T 19 380-399 33 33 33 71 71 71 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 39 39 39 81 81 81 T T T 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 42 42 42 89 89 89 F F F 27 540-559 42 42 42 92 92 92 F F F 28 560-579 41 41 41 88 88 88 F F F 29 580-599 41 41 41 88 88 88 F F F Max gen: 42, Max flush: 92 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 75 75 75 F F F 1 020-039 42 42 42 89 89 89 F F F 2 040-059 42 42 42 90 90 90 F F F 3 060-079 42 42 42 90 90 90 F F F 4 080-099 42 42 42 92 92 92 F F F 5 100-119 43 43 43 93 93 93 F F F 6 120-139 42 42 42 89 89 89 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 42 42 42 92 92 92 F F F 11 220-239 43 43 43 93 93 93 F F F 12 240-259 42 42 42 90 90 90 F F F 13 260-279 42 42 42 90 90 90 F F F 14 280-299 40 40 40 86 86 86 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 37 37 37 77 77 77 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 43 43 43 93 93 93 F F F 19 380-399 33 33 33 71 71 71 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 43 43 43 93 93 93 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 42 42 42 89 89 89 F F F 27 540-559 42 42 42 92 92 92 F F F 28 560-579 41 41 41 88 88 88 F F F 29 580-599 41 41 41 88 88 88 F F F Max gen: 43, Max flush: 93 Begin loop 22 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.737128734Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.737699672Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.737726252Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358} {"msg":"Crucible c84aefa5-8bb8-4ed0-b0d1-5cf69a41baa4 has session id: 096e7c6a-85a6-42a1-85e7-96e264cb3459","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.737857063Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Upstairs opts: Upstairs UUID: c84aefa5-8bb8-4ed0-b0d1-5cf69a41baa4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.737913164Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Crucible stats registered with UUID: c84aefa5-8bb8-4ed0-b0d1-5cf69a41baa4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.737938084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.738252648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358} {"msg":"c84aefa5-8bb8-4ed0-b0d1-5cf69a41baa4 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.73834448Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.73836325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.73837819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.73839181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.738405201Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.73841855Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.738432151Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.738496131Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.738515202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.738529292Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_connection connected from Ok(127.0.0.1:63290)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.738832696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_connection connected from Ok(127.0.0.1:59074)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.73916541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:35:17.739280801Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:35:17.739308432Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:35:17.739328782Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.739346802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.739360753Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.739447834Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.739684427Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.739710807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.739823039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.739841379Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.740254614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.740373835Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.740787421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.740810661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.740828231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.740846871Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.740860612Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:17.740873982Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.740556838Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_connection connected from Ok(127.0.0.1:36106)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.74071508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.741832754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.741871545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.742441202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.742953778Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.742977039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[0]R flush_numbers[0..12]: [75, 89, 90, 90, 92, 93, 89, 72, 82, 74, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.742992179Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[0]R generation[0..12]: [36, 42, 42, 42, 42, 43, 42, 34, 40, 36, 42, 43]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743006439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743020749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[1]R flush_numbers[0..12]: [75, 89, 90, 90, 92, 93, 89, 72, 82, 74, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743034669Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[1]R generation[0..12]: [36, 42, 42, 42, 42, 43, 42, 34, 40, 36, 42, 43]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743052899Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.74306766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[2]R flush_numbers[0..12]: [75, 89, 90, 90, 92, 93, 89, 72, 82, 74, 92, 93]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.74308161Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[2]R generation[0..12]: [36, 42, 42, 42, 42, 43, 42, 34, 40, 36, 42, 43]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.7430954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.74310966Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:35:27.74312287Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.74313763Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743150571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743163761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"":"downstairs","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743177011Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743189091Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"0","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743202551Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"1","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743215571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"client":"2","":"downstairs","session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743229212Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743241512Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"c84aefa5-8bb8-4ed0-b0d1-5cf69a41baa4 is now active with session: 096e7c6a-85a6-42a1-85e7-96e264cb3459","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743256822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743269062Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358,"session_id":"096e7c6a-85a6-42a1-85e7-96e264cb3459"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:27.743281282Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 532 len 12288 data: 4 4 4 03/30 Read block 414 len 24576 04/30 Read block 493 len 28672 05/30 Read block 304 len 4096 06/30 Write block 331 len 12288 data: 3 2 3 07/30 Write block 391 len 24576 data: 3 3 3 3 4 4 08/30 Flush 09/30 Write block 26 len 4096 data: 2 10/30 Read block 130 len 36864 11/30 Read block 571 len 40960 12/30 Read block 21 len 40960 13/30 Write block 19 len 4096 data: 6 14/30 Read block 221 len 24576 15/30 Read block 58 len 32768 16/30 Flush 17/30 Write block 244 len 40960 data: 4 4 5 6 7 7 7 8 7 7 18/30 Read block 540 len 20480 19/30 Read block 314 len 36864 20/30 Write block 371 len 28672 data: 5 5 6 8 5 6 9 21/30 Read block 432 len 4096 22/30 Write block 537 len 8192 data: 7 7 23/30 Read block 310 len 8192 24/30 Read block 333 len 4096 25/30 Read block 311 len 24576 26/30 Read block 356 len 20480 27/30 Write block 234 len 28672 data: 8 8 7 4 3 3 3 28/30 Read block 466 len 16384 29/30 Read block 164 len 12288 30/30 Read block 316 len 4096 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Read 9 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false 31 Acked 1030 Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:35:27.768938038Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 96 96 96 F F F 1 020-039 44 44 44 96 96 96 F F F 2 040-059 42 42 42 90 90 90 F F F 3 060-079 42 42 42 90 90 90 F F F 4 080-099 42 42 42 92 92 92 F F F 5 100-119 43 43 43 93 93 93 F F F 6 120-139 42 42 42 89 89 89 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 42 42 42 92 92 92 F F F 11 220-239 43 43 43 93 93 93 T T T 12 240-259 44 44 44 97 97 97 T T T 13 260-279 42 42 42 90 90 90 F F F 14 280-299 40 40 40 86 86 86 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 43 43 43 93 93 93 T T T 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 43 43 43 93 93 93 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 44 44 44 95 95 95 T T T 27 540-559 42 42 42 92 92 92 F F F 28 560-579 41 41 41 88 88 88 F F F 29 580-599 41 41 41 88 88 88 F F F Max gen: 44, Max flush: 97 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 96 96 96 F F F 1 020-039 44 44 44 96 96 96 F F F 2 040-059 42 42 42 90 90 90 F F F 3 060-079 42 42 42 90 90 90 F F F 4 080-099 42 42 42 92 92 92 F F F 5 100-119 43 43 43 93 93 93 F F F 6 120-139 42 42 42 89 89 89 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 36 36 36 74 74 74 F F F 10 200-219 42 42 42 92 92 92 F F F 11 220-239 45 45 45 98 98 98 F F F 12 240-259 45 45 45 98 98 98 F F F 13 260-279 42 42 42 90 90 90 F F F 14 280-299 40 40 40 86 86 86 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 45 45 45 98 98 98 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 43 43 43 93 93 93 F F F 23 460-479 24 24 24 48 48 48 F F F 24 480-499 21 21 21 41 41 41 F F F 25 500-519 26 26 26 51 51 51 F F F 26 520-539 45 45 45 98 98 98 F F F 27 540-559 42 42 42 92 92 92 F F F 28 560-579 41 41 41 88 88 88 F F F 29 580-599 41 41 41 88 88 88 F F F Max gen: 45, Max flush: 98 Begin loop 23 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.988305961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.988533244Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.988549284Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366} {"msg":"Crucible 8bbafbee-db8f-41b9-8191-bbdb06df661e has session id: aba7b625-afce-4145-8089-7d98285f9766","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989126631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Upstairs opts: Upstairs UUID: 8bbafbee-db8f-41b9-8191-bbdb06df661e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989180202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Crucible stats registered with UUID: 8bbafbee-db8f-41b9-8191-bbdb06df661e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989208803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989468036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366} {"msg":"8bbafbee-db8f-41b9-8191-bbdb06df661e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989532127Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989562707Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989589637Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989614648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989637038Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989659898Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989683819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989707149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989733259Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.98975861Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:35:41.989874011Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_connection connected from Ok(127.0.0.1:58434)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.989917132Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:35:41.989950072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:35:41.989981673Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.990006863Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.990034303Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_connection connected from Ok(127.0.0.1:60393)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.990060043Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.990084224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.99057598Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.99060322Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.99061813Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.990632021Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.991306839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.99133453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.991838856Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.991870176Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.991897837Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.991912427Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.991926257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:41.991939687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.990919125Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_connection connected from Ok(127.0.0.1:58392)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.991157748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.992205621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.992231671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.992802949Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993403846Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993426477Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[0]R flush_numbers[0..12]: [96, 96, 90, 90, 92, 93, 89, 72, 82, 74, 92, 98]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993444287Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[0]R generation[0..12]: [44, 44, 42, 42, 42, 43, 42, 34, 40, 36, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993461777Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993481277Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[1]R flush_numbers[0..12]: [96, 96, 90, 90, 92, 93, 89, 72, 82, 74, 92, 98]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993498827Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[1]R generation[0..12]: [44, 44, 42, 42, 42, 43, 42, 34, 40, 36, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993521658Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993540198Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[2]R flush_numbers[0..12]: [96, 96, 90, 90, 92, 93, 89, 72, 82, 74, 92, 98]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993557748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[2]R generation[0..12]: [44, 44, 42, 42, 42, 43, 42, 34, 40, 36, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993574898Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:35:51.993592379Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993611309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993628159Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993644369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993660609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"":"downstairs","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.99367752Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.99369306Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"0","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.99370972Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"1","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.99372611Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"client":"2","":"downstairs","session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.99374279Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993758281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"8bbafbee-db8f-41b9-8191-bbdb06df661e is now active with session: aba7b625-afce-4145-8089-7d98285f9766","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993778241Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993793381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366,"session_id":"aba7b625-afce-4145-8089-7d98285f9766"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:35:51.993808881Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 499 len 32768 data: 5 5 5 7 7 7 5 5 02/30 Read block 510 len 8192 03/30 Read block 458 len 16384 04/30 Write block 473 len 36864 data: 3 3 3 5 5 5 3 2 2 05/30 Read block 44 len 8192 06/30 Read block 431 len 8192 07/30 Read block 464 len 24576 08/30 Write block 570 len 40960 data: 2 2 2 3 2 2 3 3 5 5 09/30 Read block 390 len 12288 10/30 Write block 451 len 24576 data: 5 5 5 3 3 3 11/30 Read block 329 len 8192 12/30 Read block 51 len 40960 13/30 Write block 542 len 12288 data: 5 6 8 14/30 Write block 231 len 36864 data: 5 4 6 9 9 8 5 4 4 15/30 Flush 16/30 Write block 492 len 40960 data: 2 3 4 4 5 5 5 6 6 6 17/30 Read block 210 len 16384 18/30 Write block 498 len 16384 data: 6 7 7 7 19/30 Write block 41 len 16384 data: 5 5 5 4 20/30 Read block 210 len 20480 21/30 Write block 594 len 4096 data: 4 22/30 Read block 62 len 32768 23/30 Read block 586 len 32768 24/30 Write block 554 len 8192 data: 6 5 25/30 Write block 265 len 8192 data: 3 3 26/30 Read block 18 len 28672 27/30 Read block 286 len 4096 28/30 Write block 193 len 4096 data: 5 29/30 Write block 224 len 20480 data: 5 5 6 7 7 30/30 Write block 452 len 16384 data: 6 6 4 4 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 4 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Write 5 Done Sent Done false 31 Acked 1030 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 2 1 4 Done 11 10 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:35:52.019288515Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 96 96 96 F F F 1 020-039 44 44 44 96 96 96 F F F 2 040-059 42 42 42 90 90 90 T T T 3 060-079 42 42 42 90 90 90 F F F 4 080-099 42 42 42 92 92 92 F F F 5 100-119 43 43 43 93 93 93 F F F 6 120-139 42 42 42 89 89 89 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 36 36 36 74 74 74 T T T 10 200-219 42 42 42 92 92 92 F F F 11 220-239 46 46 46 99 99 99 T T T 12 240-259 45 45 45 98 98 98 F F F 13 260-279 42 42 42 90 90 90 T T T 14 280-299 40 40 40 86 86 86 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 45 45 45 98 98 98 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 46 46 46 99 99 99 T T T 23 460-479 46 46 46 99 99 99 F F F 24 480-499 46 46 46 100 100 100 F F F 25 500-519 46 46 46 100 100 100 F F F 26 520-539 45 45 45 98 98 98 F F F 27 540-559 46 46 46 99 99 99 T T T 28 560-579 46 46 46 99 99 99 F F F 29 580-599 41 41 41 88 88 88 T T T Max gen: 46, Max flush: 100 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 96 96 96 F F F 1 020-039 44 44 44 96 96 96 F F F 2 040-059 47 47 47 101 101 101 F F F 3 060-079 42 42 42 90 90 90 F F F 4 080-099 42 42 42 92 92 92 F F F 5 100-119 43 43 43 93 93 93 F F F 6 120-139 42 42 42 89 89 89 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 47 47 47 101 101 101 F F F 10 200-219 42 42 42 92 92 92 F F F 11 220-239 47 47 47 101 101 101 F F F 12 240-259 45 45 45 98 98 98 F F F 13 260-279 47 47 47 101 101 101 F F F 14 280-299 40 40 40 86 86 86 F F F 15 300-319 32 32 32 66 66 66 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 36 36 36 74 74 74 F F F 18 360-379 45 45 45 98 98 98 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 47 47 47 101 101 101 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 46 46 46 100 100 100 F F F 25 500-519 46 46 46 100 100 100 F F F 26 520-539 45 45 45 98 98 98 F F F 27 540-559 47 47 47 101 101 101 F F F 28 560-579 46 46 46 99 99 99 F F F 29 580-599 47 47 47 101 101 101 F F F Max gen: 47, Max flush: 101 Begin loop 24 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.274905358Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.275318494Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.275351714Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374} {"msg":"Crucible 48bbb6f7-1d59-4e89-9547-4b6405c43aac has session id: 62c06495-0c68-477e-8217-f817ed16cef9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.275596067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Upstairs opts: Upstairs UUID: 48bbb6f7-1d59-4e89-9547-4b6405c43aac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.275640007Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Crucible stats registered with UUID: 48bbb6f7-1d59-4e89-9547-4b6405c43aac","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.275663198Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.275958022Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374} {"msg":"48bbb6f7-1d59-4e89-9547-4b6405c43aac active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276087493Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276106313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276121094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276134924Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276148534Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276161944Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276175874Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276201335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276215735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276229465Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:06.276373877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:06.276420268Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37902)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276448538Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58462)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276475448Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:06.27658992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.27662065Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.27664372Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.276665651Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.277189687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.277216667Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.277236208Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.277259518Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.277826455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.277854476Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.278413363Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.278440423Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.278454903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.278467463Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.278481424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:06.278494934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.277703034Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34478)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.277831875Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.279326744Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.279361595Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.279833051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280372098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280393758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[0]R flush_numbers[0..12]: [96, 96, 101, 90, 92, 93, 89, 72, 82, 101, 92, 101]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280413738Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[0]R generation[0..12]: [44, 44, 47, 42, 42, 43, 42, 34, 40, 47, 42, 47]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280427908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280442468Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[1]R flush_numbers[0..12]: [96, 96, 101, 90, 92, 93, 89, 72, 82, 101, 92, 101]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280460729Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[1]R generation[0..12]: [44, 44, 47, 42, 42, 43, 42, 34, 40, 47, 42, 47]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280474939Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280489579Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[2]R flush_numbers[0..12]: [96, 96, 101, 90, 92, 93, 89, 72, 82, 101, 92, 101]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280503519Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[2]R generation[0..12]: [44, 44, 47, 42, 42, 43, 42, 34, 40, 47, 42, 47]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.28051728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.28053167Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.28054458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.28056086Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.28057464Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.28058771Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"":"downstairs","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.28060083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280612901Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"0","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280626571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"1","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280640101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"client":"2","":"downstairs","session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280653761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280668721Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"48bbb6f7-1d59-4e89-9547-4b6405c43aac is now active with session: 62c06495-0c68-477e-8217-f817ed16cef9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280681851Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280693872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374,"session_id":"62c06495-0c68-477e-8217-f817ed16cef9"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:16.280705982Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 72 len 16384 data: 3 4 3 3 02/30 Read block 320 len 8192 03/30 Write block 530 len 12288 data: 6 6 5 04/30 Read block 584 len 32768 05/30 Write block 84 len 28672 data: 3 3 3 4 4 5 5 06/30 Write block 43 len 16384 data: 6 5 4 4 07/30 Write block 357 len 20480 data: 4 4 5 4 5 08/30 Write block 307 len 16384 data: 3 3 3 3 09/30 Read block 445 len 16384 10/30 Read block 162 len 20480 11/30 Read block 455 len 36864 12/30 Read block 253 len 20480 13/30 Write block 55 len 28672 data: 5 6 4 4 4 4 5 14/30 Flush 15/30 Flush 16/30 Write block 581 len 4096 data: 3 17/30 Read block 563 len 28672 18/30 Write block 574 len 40960 data: 3 3 4 4 6 6 5 4 4 3 19/30 Read block 174 len 36864 20/30 Read block 589 len 24576 21/30 Read block 377 len 24576 22/30 Write block 524 len 16384 data: 2 3 7 7 23/30 Read block 581 len 28672 24/30 Read block 589 len 20480 25/30 Read block 97 len 4096 26/30 Read block 389 len 12288 27/30 Write block 241 len 40960 data: 3 2 3 5 5 6 7 8 8 8 28/30 Read block 418 len 20480 29/30 Write block 441 len 28672 data: 9 7 7 6 5 5 6 30/30 Write block 5 len 20480 data: 6 4 3 3 4 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 7 Done Done Done false 31 Acked 1030 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:16.307165958Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 96 96 96 T T T 1 020-039 44 44 44 96 96 96 F F F 2 040-059 48 48 48 102 102 102 F F F 3 060-079 48 48 48 102 102 102 F F F 4 080-099 48 48 48 102 102 102 F F F 5 100-119 43 43 43 93 93 93 F F F 6 120-139 42 42 42 89 89 89 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 47 47 47 101 101 101 F F F 10 200-219 42 42 42 92 92 92 F F F 11 220-239 47 47 47 101 101 101 F F F 12 240-259 45 45 45 98 98 98 T T T 13 260-279 47 47 47 101 101 101 F F F 14 280-299 40 40 40 86 86 86 F F F 15 300-319 48 48 48 102 102 102 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 48 48 48 102 102 102 F F F 18 360-379 48 48 48 102 102 102 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 47 47 47 101 101 101 T T T 23 460-479 46 46 46 99 99 99 F F F 24 480-499 46 46 46 100 100 100 F F F 25 500-519 46 46 46 100 100 100 F F F 26 520-539 48 48 48 102 102 102 T T T 27 540-559 47 47 47 101 101 101 F F F 28 560-579 48 48 48 104 104 104 F F F 29 580-599 48 48 48 104 104 104 F F F Max gen: 48, Max flush: 104 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 105 105 105 F F F 1 020-039 44 44 44 96 96 96 F F F 2 040-059 48 48 48 102 102 102 F F F 3 060-079 48 48 48 102 102 102 F F F 4 080-099 48 48 48 102 102 102 F F F 5 100-119 43 43 43 93 93 93 F F F 6 120-139 42 42 42 89 89 89 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 47 47 47 101 101 101 F F F 10 200-219 42 42 42 92 92 92 F F F 11 220-239 47 47 47 101 101 101 F F F 12 240-259 49 49 49 105 105 105 F F F 13 260-279 47 47 47 101 101 101 F F F 14 280-299 40 40 40 86 86 86 F F F 15 300-319 48 48 48 102 102 102 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 48 48 48 102 102 102 F F F 18 360-379 48 48 48 102 102 102 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 49 49 49 105 105 105 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 46 46 46 100 100 100 F F F 25 500-519 46 46 46 100 100 100 F F F 26 520-539 49 49 49 105 105 105 F F F 27 540-559 47 47 47 101 101 101 F F F 28 560-579 48 48 48 104 104 104 F F F 29 580-599 48 48 48 104 104 104 F F F Max gen: 49, Max flush: 105 Begin loop 25 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.522421349Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.522761214Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.522788524Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382} {"msg":"Crucible 65040e86-91cd-4532-ba90-0f6b413ca3e1 has session id: a99079f5-87a3-4db0-bf9c-21159d962210","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523091028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Upstairs opts: Upstairs UUID: 65040e86-91cd-4532-ba90-0f6b413ca3e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523135588Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Crucible stats registered with UUID: 65040e86-91cd-4532-ba90-0f6b413ca3e1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523155149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523524173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382} {"msg":"65040e86-91cd-4532-ba90-0f6b413ca3e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523549984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523563314Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523577714Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523591264Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523604734Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523617934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523635485Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523665675Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523682045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523696215Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_connection connected from Ok(127.0.0.1:57485)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.523826437Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_connection connected from Ok(127.0.0.1:58495)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.52403037Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:30.52407287Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:30.524102281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:30.524130541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.524152701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.524176932Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.524256553Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.524741049Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.524768549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.524784039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.524801989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.525344076Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.525478008Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.525863763Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.525887013Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.525901194Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.525913824Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.525927504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:30.525940784Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.525214305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_connection connected from Ok(127.0.0.1:46838)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.525369347Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.526800305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.526827575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.527337242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.527845408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.527863818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[0]R flush_numbers[0..12]: [105, 96, 102, 102, 102, 93, 89, 72, 82, 101, 92, 101]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.527885889Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[0]R generation[0..12]: [49, 44, 48, 48, 48, 43, 42, 34, 40, 47, 42, 47]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.527914809Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.527930209Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[1]R flush_numbers[0..12]: [105, 96, 102, 102, 102, 93, 89, 72, 82, 101, 92, 101]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.52794866Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[1]R generation[0..12]: [49, 44, 48, 48, 48, 43, 42, 34, 40, 47, 42, 47]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.52796292Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.52797734Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[2]R flush_numbers[0..12]: [105, 96, 102, 102, 102, 93, 89, 72, 82, 101, 92, 101]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.52799132Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[2]R generation[0..12]: [49, 44, 48, 48, 48, 43, 42, 34, 40, 47, 42, 47]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.52800512Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.52801947Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528032401Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528045981Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528062531Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528075951Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"":"downstairs","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528089231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528101381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"0","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528114711Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"1","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528127912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"client":"2","":"downstairs","session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528143442Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528155922Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"65040e86-91cd-4532-ba90-0f6b413ca3e1 is now active with session: a99079f5-87a3-4db0-bf9c-21159d962210","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528168932Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528180822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382,"session_id":"a99079f5-87a3-4db0-bf9c-21159d962210"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:40.528193072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 119 len 40960 data: 3 4 4 3 3 3 2 2 3 3 02/30 Write block 277 len 12288 data: 4 5 5 03/30 Read block 139 len 28672 04/30 Read block 536 len 32768 05/30 Flush 06/30 Write block 363 len 36864 data: 3 3 3 3 2 4 5 6 6 07/30 Read block 128 len 28672 08/30 Read block 88 len 12288 09/30 Flush 10/30 Write block 583 len 40960 data: 4 2 2 3 3 4 4 6 7 6 11/30 Read block 577 len 28672 12/30 Write block 262 len 12288 data: 4 3 3 13/30 Write block 248 len 4096 data: 9 14/30 Read block 491 len 8192 15/30 Read block 66 len 36864 16/30 Read block 390 len 32768 17/30 Write block 370 len 16384 data: 7 7 6 7 18/30 Write block 138 len 32768 data: 7 7 3 3 4 6 6 5 19/30 Read block 263 len 36864 20/30 Flush 21/30 Write block 293 len 36864 data: 4 4 3 2 2 2 3 3 3 22/30 Write block 354 len 40960 data: 7 7 6 5 5 6 5 6 3 4 23/30 Read block 279 len 12288 24/30 Write block 130 len 8192 data: 6 5 25/30 Read block 361 len 32768 26/30 Flush 27/30 Write block 51 len 12288 data: 6 8 7 28/30 Write block 215 len 40960 data: 3 3 3 2 3 3 5 6 5 6 29/30 Write block 490 len 20480 data: 4 4 3 4 5 30/30 Write block 41 len 28672 data: 6 6 7 6 5 5 3 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 10 Done Done Done false 30 Acked 1029 Write 5 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:40.558884342Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 105 105 105 F F F 1 020-039 44 44 44 96 96 96 F F F 2 040-059 48 48 48 102 102 102 T T T 3 060-079 48 48 48 102 102 102 F F F 4 080-099 48 48 48 102 102 102 F F F 5 100-119 50 50 50 106 106 106 F F F 6 120-139 50 50 50 110 110 110 F F F 7 140-159 50 50 50 108 108 108 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 47 47 47 101 101 101 F F F 10 200-219 42 42 42 92 92 92 T T T 11 220-239 47 47 47 101 101 101 T T T 12 240-259 50 50 50 108 108 108 F F F 13 260-279 50 50 50 108 108 108 F F F 14 280-299 50 50 50 110 110 110 F F F 15 300-319 50 50 50 110 110 110 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 50 50 50 110 110 110 F F F 18 360-379 50 50 50 110 110 110 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 49 49 49 105 105 105 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 46 46 46 100 100 100 T T T 25 500-519 46 46 46 100 100 100 F F F 26 520-539 49 49 49 105 105 105 F F F 27 540-559 47 47 47 101 101 101 F F F 28 560-579 48 48 48 104 104 104 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 50, Max flush: 110 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 105 105 105 F F F 1 020-039 44 44 44 96 96 96 F F F 2 040-059 51 51 51 111 111 111 F F F 3 060-079 48 48 48 102 102 102 F F F 4 080-099 48 48 48 102 102 102 F F F 5 100-119 50 50 50 106 106 106 F F F 6 120-139 50 50 50 110 110 110 F F F 7 140-159 50 50 50 108 108 108 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 47 47 47 101 101 101 F F F 10 200-219 51 51 51 111 111 111 F F F 11 220-239 51 51 51 111 111 111 F F F 12 240-259 50 50 50 108 108 108 F F F 13 260-279 50 50 50 108 108 108 F F F 14 280-299 50 50 50 110 110 110 F F F 15 300-319 50 50 50 110 110 110 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 50 50 50 110 110 110 F F F 18 360-379 50 50 50 110 110 110 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 49 49 49 105 105 105 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 51 51 51 111 111 111 F F F 25 500-519 46 46 46 100 100 100 F F F 26 520-539 49 49 49 105 105 105 F F F 27 540-559 47 47 47 101 101 101 F F F 28 560-579 48 48 48 104 104 104 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 51, Max flush: 111 Begin loop 26 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.778971735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.779258638Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.779288069Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390} {"msg":"Crucible 90245c7c-047c-4d89-a290-4b31ffaf56c4 has session id: 146b6638-297c-44f4-9392-292cebbd0d83","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.779662934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Upstairs opts: Upstairs UUID: 90245c7c-047c-4d89-a290-4b31ffaf56c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.779709014Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Crucible stats registered with UUID: 90245c7c-047c-4d89-a290-4b31ffaf56c4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.779733084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780056919Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390} {"msg":"90245c7c-047c-4d89-a290-4b31ffaf56c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.78015811Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.78018341Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.78019961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780213841Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780227741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780241351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780255061Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780279961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780294671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780308612Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_connection connected from Ok(127.0.0.1:42944)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780471684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_connection connected from Ok(127.0.0.1:47151)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780634426Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:54.780674566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:54.780694657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:36:54.780711197Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780723937Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780737237Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.780835528Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.781347495Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.781380215Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.781406546Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.781432676Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.782048984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.782078454Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.78255272Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.7825801Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.782595001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.782613911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.782627741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:36:54.782641271Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.782472319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_connection connected from Ok(127.0.0.1:55468)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.782781313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.783617873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.783645214Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.78414541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784682657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784705867Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[0]R flush_numbers[0..12]: [105, 96, 111, 102, 102, 106, 110, 108, 82, 101, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784721238Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[0]R generation[0..12]: [49, 44, 51, 48, 48, 50, 50, 50, 40, 47, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784735878Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784750258Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[1]R flush_numbers[0..12]: [105, 96, 111, 102, 102, 106, 110, 108, 82, 101, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784770048Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[1]R generation[0..12]: [49, 44, 51, 48, 48, 50, 50, 50, 40, 47, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784784438Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784798588Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[2]R flush_numbers[0..12]: [105, 96, 111, 102, 102, 106, 110, 108, 82, 101, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784812469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[2]R generation[0..12]: [49, 44, 51, 48, 48, 50, 50, 50, 40, 47, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784826219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784840389Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784856829Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784870309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.78488353Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.78489673Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"":"downstairs","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.78491004Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.78492217Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"0","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.7849358Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"1","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.78494909Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"client":"2","":"downstairs","session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784971911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.784991881Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"90245c7c-047c-4d89-a290-4b31ffaf56c4 is now active with session: 146b6638-297c-44f4-9392-292cebbd0d83","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.785027262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.785039742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390,"session_id":"146b6638-297c-44f4-9392-292cebbd0d83"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:04.785052062Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 35 len 20480 data: 2 2 2 2 4 02/30 Read block 524 len 32768 03/30 Write block 38 len 20480 data: 3 5 5 7 7 04/30 Read block 340 len 28672 05/30 Write block 30 len 36864 data: 3 3 2 2 2 3 3 3 4 06/30 Read block 20 len 20480 07/30 Write block 567 len 8192 data: 3 3 08/30 Write block 313 len 12288 data: 2 3 3 09/30 Write block 27 len 36864 data: 2 4 5 4 4 3 3 3 4 10/30 Read block 478 len 28672 11/30 Read block 310 len 40960 12/30 Read block 169 len 32768 13/30 Write block 446 len 4096 data: 6 14/30 Read block 398 len 4096 15/30 Read block 93 len 36864 16/30 Read block 382 len 4096 17/30 Read block 580 len 36864 18/30 Read block 513 len 8192 19/30 Read block 236 len 16384 20/30 Write block 500 len 4096 data: 8 21/30 Read block 185 len 8192 22/30 Write block 196 len 36864 data: 5 7 5 3 3 3 4 4 4 23/30 Read block 190 len 20480 24/30 Write block 132 len 16384 data: 5 4 5 6 25/30 Read block 82 len 8192 26/30 Write block 481 len 12288 data: 3 2 2 27/30 Read block 295 len 24576 28/30 Read block 123 len 20480 29/30 Read block 586 len 40960 30/30 Read block 309 len 8192 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false 31 Acked 1030 Read 2 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 1 2 Done 10 11 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:37:04.809749445Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 105 105 105 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 52 52 52 112 112 112 F F F 3 060-079 48 48 48 102 102 102 F F F 4 080-099 48 48 48 102 102 102 F F F 5 100-119 50 50 50 106 106 106 F F F 6 120-139 50 50 50 110 110 110 T T T 7 140-159 50 50 50 108 108 108 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 47 47 47 101 101 101 T T T 10 200-219 51 51 51 111 111 111 T T T 11 220-239 51 51 51 111 111 111 F F F 12 240-259 50 50 50 108 108 108 F F F 13 260-279 50 50 50 108 108 108 F F F 14 280-299 50 50 50 110 110 110 F F F 15 300-319 52 52 52 112 112 112 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 50 50 50 110 110 110 F F F 18 360-379 50 50 50 110 110 110 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 52 52 52 112 112 112 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 51 51 51 111 111 111 T T T 25 500-519 46 46 46 100 100 100 T T T 26 520-539 49 49 49 105 105 105 F F F 27 540-559 47 47 47 101 101 101 F F F 28 560-579 52 52 52 112 112 112 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 52, Max flush: 112 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 105 105 105 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 52 52 52 112 112 112 F F F 3 060-079 48 48 48 102 102 102 F F F 4 080-099 48 48 48 102 102 102 F F F 5 100-119 50 50 50 106 106 106 F F F 6 120-139 53 53 53 113 113 113 F F F 7 140-159 50 50 50 108 108 108 F F F 8 160-179 40 40 40 82 82 82 F F F 9 180-199 53 53 53 113 113 113 F F F 10 200-219 53 53 53 113 113 113 F F F 11 220-239 51 51 51 111 111 111 F F F 12 240-259 50 50 50 108 108 108 F F F 13 260-279 50 50 50 108 108 108 F F F 14 280-299 50 50 50 110 110 110 F F F 15 300-319 52 52 52 112 112 112 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 50 50 50 110 110 110 F F F 18 360-379 50 50 50 110 110 110 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 40 40 40 84 84 84 F F F 21 420-439 39 39 39 81 81 81 F F F 22 440-459 52 52 52 112 112 112 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 53 53 53 113 113 113 F F F 25 500-519 53 53 53 113 113 113 F F F 26 520-539 49 49 49 105 105 105 F F F 27 540-559 47 47 47 101 101 101 F F F 28 560-579 52 52 52 112 112 112 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 53, Max flush: 113 Begin loop 27 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.035372659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.035640242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.035666342Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398} {"msg":"Crucible 8591a1eb-1901-4c2d-900c-8646c48d09c2 has session id: 06dc326a-908c-4f65-a63e-9967315aeda0","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036129068Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Upstairs opts: Upstairs UUID: 8591a1eb-1901-4c2d-900c-8646c48d09c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036174629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Crucible stats registered with UUID: 8591a1eb-1901-4c2d-900c-8646c48d09c2","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036198509Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036506863Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398} {"msg":"8591a1eb-1901-4c2d-900c-8646c48d09c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036531513Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036544634Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036559114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036572724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036586424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036599794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036613354Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036673915Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036692325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.036710085Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:37:19.036890948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:37:19.036932408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_connection connected from Ok(127.0.0.1:64190)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.03703787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:37:19.03707659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.03709963Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.037122891Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_connection connected from Ok(127.0.0.1:34327)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.037149001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.037177411Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.037768759Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.03780665Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.03783612Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.03786301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.038421417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.038559379Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.038955634Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.038979224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.038993685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.039064886Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.039083846Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:19.039097466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.038099323Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40724)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.038262545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.039523771Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.039555481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040255461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040803937Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040824488Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[0]R flush_numbers[0..12]: [105, 112, 112, 102, 102, 106, 113, 108, 82, 113, 113, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040840028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[0]R generation[0..12]: [49, 52, 52, 48, 48, 50, 53, 50, 40, 53, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040854288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040873498Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[1]R flush_numbers[0..12]: [105, 112, 112, 102, 102, 106, 113, 108, 82, 113, 113, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040888309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[1]R generation[0..12]: [49, 52, 52, 48, 48, 50, 53, 50, 40, 53, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040902549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040916789Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[2]R flush_numbers[0..12]: [105, 112, 112, 102, 102, 106, 113, 108, 82, 113, 113, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040930819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[2]R generation[0..12]: [49, 52, 52, 48, 48, 50, 53, 50, 40, 53, 53, 51]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040944549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040958929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.040972019Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.04098921Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.04100242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.04101562Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"":"downstairs","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.04102922Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.041041231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"0","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.04105486Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"1","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.041070661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"client":"2","":"downstairs","session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.041084471Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.041096951Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"8591a1eb-1901-4c2d-900c-8646c48d09c2 is now active with session: 06dc326a-908c-4f65-a63e-9967315aeda0","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.041109941Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.041122232Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398,"session_id":"06dc326a-908c-4f65-a63e-9967315aeda0"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:29.041134552Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 356 len 12288 data: 7 6 6 02/30 Write block 376 len 40960 data: 7 10 8 7 6 6 6 3 3 3 03/30 Read block 328 len 40960 04/30 Write block 457 len 4096 data: 3 05/30 Read block 570 len 16384 06/30 Read block 378 len 8192 07/30 Flush 08/30 Write block 546 len 20480 data: 4 4 5 8 10 09/30 Read block 527 len 16384 10/30 Read block 456 len 36864 11/30 Read block 100 len 12288 12/30 Write block 403 len 32768 data: 3 2 3 3 4 4 4 4 13/30 Write block 281 len 16384 data: 4 4 4 3 14/30 Write block 199 len 8192 data: 4 4 15/30 Read block 415 len 4096 16/30 Write block 396 len 24576 data: 5 5 5 5 5 4 17/30 Flush 18/30 Write block 482 len 16384 data: 3 3 3 4 19/30 Read block 454 len 8192 20/30 Write block 566 len 28672 data: 2 4 4 2 3 3 3 21/30 Read block 229 len 4096 22/30 Read block 287 len 20480 23/30 Write block 251 len 32768 data: 9 8 8 8 9 6 6 6 24/30 Write block 248 len 16384 data: 10 9 9 10 25/30 Write block 369 len 12288 data: 6 8 8 26/30 Write block 418 len 32768 data: 5 4 4 4 5 6 6 6 27/30 Write block 526 len 36864 data: 8 8 7 6 7 7 6 5 5 28/30 Write block 166 len 36864 data: 4 5 5 4 4 4 5 6 7 29/30 Read block 182 len 12288 30/30 Read block 455 len 36864 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 4 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Write 7 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 12 12 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:37:29.066692366Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 105 105 105 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 52 52 52 112 112 112 F F F 3 060-079 48 48 48 102 102 102 F F F 4 080-099 48 48 48 102 102 102 F F F 5 100-119 50 50 50 106 106 106 F F F 6 120-139 53 53 53 113 113 113 F F F 7 140-159 50 50 50 108 108 108 F F F 8 160-179 40 40 40 82 82 82 T T T 9 180-199 54 54 54 115 115 115 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 51 51 51 111 111 111 F F F 12 240-259 50 50 50 108 108 108 T T T 13 260-279 50 50 50 108 108 108 F F F 14 280-299 54 54 54 115 115 115 F F F 15 300-319 52 52 52 112 112 112 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 54 54 54 114 114 114 F F F 18 360-379 54 54 54 114 114 114 T T T 19 380-399 54 54 54 115 115 115 F F F 20 400-419 54 54 54 115 115 115 T T T 21 420-439 39 39 39 81 81 81 T T T 22 440-459 54 54 54 114 114 114 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 53 53 53 113 113 113 T T T 25 500-519 53 53 53 113 113 113 F F F 26 520-539 49 49 49 105 105 105 T T T 27 540-559 54 54 54 115 115 115 F F F 28 560-579 52 52 52 112 112 112 T T T 29 580-599 50 50 50 108 108 108 F F F Max gen: 54, Max flush: 115 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 105 105 105 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 52 52 52 112 112 112 F F F 3 060-079 48 48 48 102 102 102 F F F 4 080-099 48 48 48 102 102 102 F F F 5 100-119 50 50 50 106 106 106 F F F 6 120-139 53 53 53 113 113 113 F F F 7 140-159 50 50 50 108 108 108 F F F 8 160-179 55 55 55 116 116 116 F F F 9 180-199 54 54 54 115 115 115 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 51 51 51 111 111 111 F F F 12 240-259 55 55 55 116 116 116 F F F 13 260-279 50 50 50 108 108 108 F F F 14 280-299 54 54 54 115 115 115 F F F 15 300-319 52 52 52 112 112 112 F F F 16 320-339 44 44 44 95 95 95 F F F 17 340-359 54 54 54 114 114 114 F F F 18 360-379 55 55 55 116 116 116 F F F 19 380-399 54 54 54 115 115 115 F F F 20 400-419 55 55 55 116 116 116 F F F 21 420-439 55 55 55 116 116 116 F F F 22 440-459 54 54 54 114 114 114 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 55 55 55 116 116 116 F F F 25 500-519 53 53 53 113 113 113 F F F 26 520-539 55 55 55 116 116 116 F F F 27 540-559 54 54 54 115 115 115 F F F 28 560-579 55 55 55 116 116 116 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 55, Max flush: 116 Begin loop 28 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.335173453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.335490207Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.335511687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406} {"msg":"Crucible 6323cdf5-cae6-4398-88c1-cc1b4eb2562e has session id: 0f34706c-bd45-4f1c-a3be-2409aa128b1c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.335935323Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Upstairs opts: Upstairs UUID: 6323cdf5-cae6-4398-88c1-cc1b4eb2562e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.335980733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Crucible stats registered with UUID: 6323cdf5-cae6-4398-88c1-cc1b4eb2562e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.336003654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.336376128Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406} {"msg":"6323cdf5-cae6-4398-88c1-cc1b4eb2562e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.336444299Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.336462629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.3364773Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.33649103Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.33650468Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.33651785Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.33653131Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.336560711Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.336576631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.336590761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57590)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.336827334Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:37:43.336926415Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33925)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.336963236Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:37:43.336992866Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:37:43.337022047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.337045377Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.337070687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.337094017Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.337649415Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.337675285Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.337689865Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.337711265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.338345263Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.338477345Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.338814419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.33885203Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.33888041Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.33890672Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.338934051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:43.338956811Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.338739669Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43338)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.339157624Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.339850973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.339890743Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.34041773Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.340928766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.340952006Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[0]R flush_numbers[0..12]: [105, 112, 112, 102, 102, 106, 113, 108, 116, 115, 115, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.340973357Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[0]R generation[0..12]: [49, 52, 52, 48, 48, 50, 53, 50, 55, 54, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.340990077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341005127Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[1]R flush_numbers[0..12]: [105, 112, 112, 102, 102, 106, 113, 108, 116, 115, 115, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341019267Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[1]R generation[0..12]: [49, 52, 52, 48, 48, 50, 53, 50, 55, 54, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341032898Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341047138Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[2]R flush_numbers[0..12]: [105, 112, 112, 102, 102, 106, 113, 108, 116, 115, 115, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341061118Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[2]R generation[0..12]: [49, 52, 52, 48, 48, 50, 53, 50, 55, 54, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341074828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341088868Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341102008Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341119329Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341132439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341145389Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"":"downstairs","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341158559Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341173139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"0","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341186529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"1","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.3411998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"client":"2","":"downstairs","session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.34121332Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.34122564Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"6323cdf5-cae6-4398-88c1-cc1b4eb2562e is now active with session: 0f34706c-bd45-4f1c-a3be-2409aa128b1c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.34123873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.34125063Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406,"session_id":"0f34706c-bd45-4f1c-a3be-2409aa128b1c"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:37:53.341262861Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 74 len 40960 data: 4 4 3 3 3 3 2 2 2 3 02/30 Flush 03/30 Write block 165 len 16384 data: 4 5 6 6 04/30 Write block 42 len 24576 data: 8 8 7 6 6 4 05/30 Read block 210 len 24576 06/30 Write block 187 len 4096 data: 5 07/30 Flush 08/30 Flush 09/30 Write block 65 len 16384 data: 5 4 4 4 10/30 Read block 423 len 8192 11/30 Write block 322 len 24576 data: 7 6 5 6 5 5 12/30 Write block 438 len 28672 data: 4 6 6 10 8 8 7 13/30 Write block 65 len 24576 data: 6 5 5 5 4 4 14/30 Write block 531 len 8192 data: 8 7 15/30 Read block 103 len 4096 16/30 Read block 92 len 20480 17/30 Read block 434 len 24576 18/30 Write block 498 len 36864 data: 7 8 9 8 8 8 8 6 6 19/30 Read block 71 len 4096 20/30 Read block 429 len 20480 21/30 Read block 515 len 24576 22/30 Read block 377 len 12288 23/30 Read block 206 len 20480 24/30 Write block 224 len 8192 data: 7 6 25/30 Write block 428 len 24576 data: 6 6 5 5 4 4 26/30 Read block 368 len 28672 27/30 Read block 132 len 36864 28/30 Read block 497 len 12288 29/30 Write block 148 len 8192 data: 7 7 30/30 Read block 75 len 20480 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 2 Done Done Done false 31 Acked 1030 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:37:53.36798445Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 105 105 105 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 56 56 56 120 120 120 F F F 4 080-099 56 56 56 117 117 117 F F F 5 100-119 50 50 50 106 106 106 F F F 6 120-139 53 53 53 113 113 113 F F F 7 140-159 50 50 50 108 108 108 T T T 8 160-179 56 56 56 118 118 118 F F F 9 180-199 56 56 56 118 118 118 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 51 51 51 111 111 111 T T T 12 240-259 55 55 55 116 116 116 F F F 13 260-279 50 50 50 108 108 108 F F F 14 280-299 54 54 54 115 115 115 F F F 15 300-319 52 52 52 112 112 112 F F F 16 320-339 56 56 56 120 120 120 F F F 17 340-359 54 54 54 114 114 114 F F F 18 360-379 55 55 55 116 116 116 F F F 19 380-399 54 54 54 115 115 115 F F F 20 400-419 55 55 55 116 116 116 F F F 21 420-439 56 56 56 120 120 120 T T T 22 440-459 56 56 56 120 120 120 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 56 56 56 120 120 120 F F F 25 500-519 56 56 56 120 120 120 F F F 26 520-539 56 56 56 120 120 120 F F F 27 540-559 54 54 54 115 115 115 F F F 28 560-579 55 55 55 116 116 116 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 56, Max flush: 120 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 105 105 105 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 56 56 56 120 120 120 F F F 4 080-099 56 56 56 117 117 117 F F F 5 100-119 50 50 50 106 106 106 F F F 6 120-139 53 53 53 113 113 113 F F F 7 140-159 57 57 57 121 121 121 F F F 8 160-179 56 56 56 118 118 118 F F F 9 180-199 56 56 56 118 118 118 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 57 57 57 121 121 121 F F F 12 240-259 55 55 55 116 116 116 F F F 13 260-279 50 50 50 108 108 108 F F F 14 280-299 54 54 54 115 115 115 F F F 15 300-319 52 52 52 112 112 112 F F F 16 320-339 56 56 56 120 120 120 F F F 17 340-359 54 54 54 114 114 114 F F F 18 360-379 55 55 55 116 116 116 F F F 19 380-399 54 54 54 115 115 115 F F F 20 400-419 55 55 55 116 116 116 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 56 56 56 120 120 120 F F F 23 460-479 46 46 46 99 99 99 F F F 24 480-499 56 56 56 120 120 120 F F F 25 500-519 56 56 56 120 120 120 F F F 26 520-539 56 56 56 120 120 120 F F F 27 540-559 54 54 54 115 115 115 F F F 28 560-579 55 55 55 116 116 116 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 57, Max flush: 121 Begin loop 29 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.574794994Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.57522487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.57525118Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414} {"msg":"Crucible 5125dcb8-0e72-4d90-9fed-037468cb1491 has session id: a539ade9-4be9-4ea0-abd4-5a17aae26425","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.575580994Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Upstairs opts: Upstairs UUID: 5125dcb8-0e72-4d90-9fed-037468cb1491, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.575641345Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Crucible stats registered with UUID: 5125dcb8-0e72-4d90-9fed-037468cb1491","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.575658875Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.575974609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414} {"msg":"5125dcb8-0e72-4d90-9fed-037468cb1491 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.57607761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576109241Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576137001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576161711Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576184992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576208922Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576236452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576254352Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576277883Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576301973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:07.576589277Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:07.576618337Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_connection connected from Ok(127.0.0.1:52639)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576634347Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:07.576650877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576663368Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576676668Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576706218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_connection connected from Ok(127.0.0.1:63020)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.576786229Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.577205764Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.577228625Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.577340656Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.577380837Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.577682241Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.577774272Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.578173877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.578201257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.578216788Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.578314619Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.578334889Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:07.578348849Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.577706931Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_connection connected from Ok(127.0.0.1:50307)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.577914294Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.579075508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.579101809Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.579657246Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580207923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580239143Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[0]R flush_numbers[0..12]: [105, 112, 118, 120, 117, 106, 113, 121, 118, 118, 115, 121]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580256003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[0]R generation[0..12]: [49, 52, 56, 56, 56, 50, 53, 57, 56, 56, 54, 57]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580270884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580285364Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[1]R flush_numbers[0..12]: [105, 112, 118, 120, 117, 106, 113, 121, 118, 118, 115, 121]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580299494Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[1]R generation[0..12]: [49, 52, 56, 56, 56, 50, 53, 57, 56, 56, 54, 57]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580313384Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580327564Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[2]R flush_numbers[0..12]: [105, 112, 118, 120, 117, 106, 113, 121, 118, 118, 115, 121]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580341624Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[2]R generation[0..12]: [49, 52, 56, 56, 56, 50, 53, 57, 56, 56, 54, 57]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580355445Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580369515Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580386475Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580400035Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580413325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580426516Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"":"downstairs","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580442326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580454606Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"0","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580468206Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"1","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580481966Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"client":"2","":"downstairs","session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580495837Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580508307Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"5125dcb8-0e72-4d90-9fed-037468cb1491 is now active with session: a539ade9-4be9-4ea0-abd4-5a17aae26425","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580522277Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580534487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414,"session_id":"a539ade9-4be9-4ea0-abd4-5a17aae26425"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:17.580546877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 312 len 8192 data: 2 3 02/30 Read block 495 len 8192 03/30 Read block 141 len 28672 04/30 Write block 5 len 16384 data: 7 5 4 4 05/30 Write block 71 len 4096 data: 2 06/30 Read block 442 len 16384 07/30 Write block 483 len 4096 data: 4 08/30 Write block 473 len 40960 data: 4 4 4 6 6 6 4 3 4 4 09/30 Read block 395 len 20480 10/30 Read block 353 len 16384 11/30 Read block 94 len 4096 12/30 Flush 13/30 Write block 518 len 4096 data: 2 14/30 Read block 128 len 28672 15/30 Read block 171 len 16384 16/30 Read block 298 len 8192 17/30 Write block 237 len 12288 data: 6 5 5 18/30 Read block 466 len 8192 19/30 Flush 20/30 Write block 302 len 28672 data: 3 4 3 3 3 4 4 21/30 Read block 190 len 20480 22/30 Read block 84 len 12288 23/30 Write block 370 len 4096 data: 9 24/30 Read block 4 len 20480 25/30 Write block 64 len 32768 data: 5 7 6 6 6 5 5 3 26/30 Read block 536 len 20480 27/30 Read block 34 len 12288 28/30 Read block 3 len 16384 29/30 Write block 113 len 32768 data: 3 3 4 4 3 3 4 5 30/30 Write block 520 len 28672 data: 2 2 2 2 3 4 9 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 7 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 8 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:17.604945497Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 122 122 122 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 58 58 58 122 122 122 T T T 4 080-099 56 56 56 117 117 117 F F F 5 100-119 50 50 50 106 106 106 T T T 6 120-139 53 53 53 113 113 113 T T T 7 140-159 57 57 57 121 121 121 F F F 8 160-179 56 56 56 118 118 118 F F F 9 180-199 56 56 56 118 118 118 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 58 58 58 123 123 123 F F F 12 240-259 55 55 55 116 116 116 F F F 13 260-279 50 50 50 108 108 108 F F F 14 280-299 54 54 54 115 115 115 F F F 15 300-319 58 58 58 122 122 122 T T T 16 320-339 56 56 56 120 120 120 F F F 17 340-359 54 54 54 114 114 114 F F F 18 360-379 55 55 55 116 116 116 T T T 19 380-399 54 54 54 115 115 115 F F F 20 400-419 55 55 55 116 116 116 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 56 56 56 120 120 120 F F F 23 460-479 58 58 58 122 122 122 F F F 24 480-499 58 58 58 122 122 122 F F F 25 500-519 58 58 58 123 123 123 F F F 26 520-539 56 56 56 120 120 120 T T T 27 540-559 54 54 54 115 115 115 F F F 28 560-579 55 55 55 116 116 116 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 58, Max flush: 123 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 122 122 122 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 59 59 59 124 124 124 F F F 4 080-099 56 56 56 117 117 117 F F F 5 100-119 59 59 59 124 124 124 F F F 6 120-139 59 59 59 124 124 124 F F F 7 140-159 57 57 57 121 121 121 F F F 8 160-179 56 56 56 118 118 118 F F F 9 180-199 56 56 56 118 118 118 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 58 58 58 123 123 123 F F F 12 240-259 55 55 55 116 116 116 F F F 13 260-279 50 50 50 108 108 108 F F F 14 280-299 54 54 54 115 115 115 F F F 15 300-319 59 59 59 124 124 124 F F F 16 320-339 56 56 56 120 120 120 F F F 17 340-359 54 54 54 114 114 114 F F F 18 360-379 59 59 59 124 124 124 F F F 19 380-399 54 54 54 115 115 115 F F F 20 400-419 55 55 55 116 116 116 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 56 56 56 120 120 120 F F F 23 460-479 58 58 58 122 122 122 F F F 24 480-499 58 58 58 122 122 122 F F F 25 500-519 58 58 58 123 123 123 F F F 26 520-539 59 59 59 124 124 124 F F F 27 540-559 54 54 54 115 115 115 F F F 28 560-579 55 55 55 116 116 116 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 59, Max flush: 124 Begin loop 30 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.846130657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.846489812Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.846525542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422} {"msg":"Crucible 57139576-1a65-4b87-8cb6-396a4b62e26c has session id: 048da07b-d3c7-42e2-a0df-176768b5c022","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.846867426Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Upstairs opts: Upstairs UUID: 57139576-1a65-4b87-8cb6-396a4b62e26c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.846995948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Crucible stats registered with UUID: 57139576-1a65-4b87-8cb6-396a4b62e26c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847031868Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847306452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422} {"msg":"57139576-1a65-4b87-8cb6-396a4b62e26c active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847400573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847419813Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847434714Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847449024Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847462724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847480514Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847494594Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847562905Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847582586Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847596736Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:31.847774768Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:31.847804478Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:31.847823669Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847844669Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847868489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_connection connected from Ok(127.0.0.1:58416)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.847977571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.848037161Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_connection connected from Ok(127.0.0.1:50723)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.848063542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.848523078Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.848560238Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.848592718Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.848619929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.849236696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.849264697Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.849849054Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.849875745Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.849890845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.849903995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.849917555Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:31.849930955Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.849290777Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_connection connected from Ok(127.0.0.1:37465)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.84946653Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.850725305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.850763606Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.851270592Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.851815559Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.85183996Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[0]R flush_numbers[0..12]: [122, 112, 118, 124, 117, 124, 124, 121, 118, 118, 115, 123]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.85185589Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[0]R generation[0..12]: [58, 52, 56, 59, 56, 59, 59, 57, 56, 56, 54, 58]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.85187018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.85188514Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[1]R flush_numbers[0..12]: [122, 112, 118, 124, 117, 124, 124, 121, 118, 118, 115, 123]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.85189954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[1]R generation[0..12]: [58, 52, 56, 59, 56, 59, 59, 57, 56, 56, 54, 58]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.851913341Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.851927711Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[2]R flush_numbers[0..12]: [122, 112, 118, 124, 117, 124, 124, 121, 118, 118, 115, 123]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.851941741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[2]R generation[0..12]: [58, 52, 56, 59, 56, 59, 59, 57, 56, 56, 54, 58]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.851955521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.851969821Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.851982761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.851999242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852013302Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852029892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"":"downstairs","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852043522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852055782Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"0","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852069112Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"1","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852082323Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"client":"2","":"downstairs","session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852096493Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852109113Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"57139576-1a65-4b87-8cb6-396a4b62e26c is now active with session: 048da07b-d3c7-42e2-a0df-176768b5c022","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852122073Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852134073Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422,"session_id":"048da07b-d3c7-42e2-a0df-176768b5c022"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:41.852146373Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 246 len 28672 data: 7 8 11 10 10 11 9 02/30 Write block 85 len 28672 data: 4 4 5 5 6 6 5 03/30 Read block 596 len 16384 04/30 Read block 152 len 8192 05/30 Read block 196 len 16384 06/30 Write block 260 len 16384 data: 7 5 5 4 07/30 Flush 08/30 Write block 130 len 4096 data: 7 09/30 Read block 537 len 40960 10/30 Read block 26 len 8192 11/30 Read block 585 len 24576 12/30 Flush 13/30 Write block 384 len 28672 data: 4 4 3 2 3 3 3 14/30 Read block 299 len 4096 15/30 Read block 162 len 8192 16/30 Read block 553 len 16384 17/30 Read block 190 len 32768 18/30 Write block 296 len 12288 data: 3 3 3 19/30 Write block 129 len 12288 data: 5 8 6 20/30 Write block 537 len 16384 data: 8 8 6 5 21/30 Read block 549 len 16384 22/30 Write block 112 len 28672 data: 4 4 4 5 5 4 4 23/30 Read block 50 len 24576 24/30 Write block 386 len 4096 data: 4 25/30 Write block 229 len 8192 data: 8 6 26/30 Read block 572 len 32768 27/30 Read block 598 len 8192 28/30 Read block 379 len 32768 29/30 Read block 16 len 20480 30/30 Read block 352 len 36864 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 12 12 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:41.876483662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 122 122 122 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 59 59 59 124 124 124 F F F 4 080-099 60 60 60 125 125 125 F F F 5 100-119 59 59 59 124 124 124 T T T 6 120-139 60 60 60 126 126 126 T T T 7 140-159 57 57 57 121 121 121 F F F 8 160-179 56 56 56 118 118 118 F F F 9 180-199 56 56 56 118 118 118 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 58 58 58 123 123 123 T T T 12 240-259 60 60 60 125 125 125 F F F 13 260-279 60 60 60 125 125 125 F F F 14 280-299 54 54 54 115 115 115 T T T 15 300-319 59 59 59 124 124 124 F F F 16 320-339 56 56 56 120 120 120 F F F 17 340-359 54 54 54 114 114 114 F F F 18 360-379 59 59 59 124 124 124 F F F 19 380-399 60 60 60 127 127 127 T T T 20 400-419 55 55 55 116 116 116 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 56 56 56 120 120 120 F F F 23 460-479 58 58 58 122 122 122 F F F 24 480-499 58 58 58 122 122 122 F F F 25 500-519 58 58 58 123 123 123 F F F 26 520-539 59 59 59 124 124 124 T T T 27 540-559 54 54 54 115 115 115 T T T 28 560-579 55 55 55 116 116 116 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 60, Max flush: 127 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 122 122 122 F F F 1 020-039 52 52 52 112 112 112 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 59 59 59 124 124 124 F F F 4 080-099 60 60 60 125 125 125 F F F 5 100-119 61 61 61 128 128 128 F F F 6 120-139 61 61 61 128 128 128 F F F 7 140-159 57 57 57 121 121 121 F F F 8 160-179 56 56 56 118 118 118 F F F 9 180-199 56 56 56 118 118 118 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 61 61 61 128 128 128 F F F 12 240-259 60 60 60 125 125 125 F F F 13 260-279 60 60 60 125 125 125 F F F 14 280-299 61 61 61 128 128 128 F F F 15 300-319 59 59 59 124 124 124 F F F 16 320-339 56 56 56 120 120 120 F F F 17 340-359 54 54 54 114 114 114 F F F 18 360-379 59 59 59 124 124 124 F F F 19 380-399 61 61 61 128 128 128 F F F 20 400-419 55 55 55 116 116 116 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 56 56 56 120 120 120 F F F 23 460-479 58 58 58 122 122 122 F F F 24 480-499 58 58 58 122 122 122 F F F 25 500-519 58 58 58 123 123 123 F F F 26 520-539 61 61 61 128 128 128 F F F 27 540-559 61 61 61 128 128 128 F F F 28 560-579 55 55 55 116 116 116 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 61, Max flush: 128 Begin loop 31 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.133778077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.134116522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.134135262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430} {"msg":"Crucible 68d182d2-8f0f-4b52-8ece-ff3141b29258 has session id: f7fe2dae-7cd6-434f-b32e-5530f74278ee","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.134538537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Upstairs opts: Upstairs UUID: 68d182d2-8f0f-4b52-8ece-ff3141b29258, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.134570707Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Crucible stats registered with UUID: 68d182d2-8f0f-4b52-8ece-ff3141b29258","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.134585688Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.134928272Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430} {"msg":"68d182d2-8f0f-4b52-8ece-ff3141b29258 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135025493Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135050983Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135067204Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135085754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135104404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135127024Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135145185Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135159235Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135173175Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135186945Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:56.135397058Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:56690)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.13554892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:56.13557413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:38:56.13559121Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:42596)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.13560387Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135618131Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135640011Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.135664741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.136211558Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.136237529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.136252019Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.136265489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.136849446Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.136877647Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.137424193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.137450574Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.137465164Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.137477914Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.137492034Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:38:56.137505545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.137189341Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:41177)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.137347812Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.138525027Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.138562288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139130465Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139753453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139782234Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[0]R flush_numbers[0..12]: [122, 112, 118, 124, 125, 128, 128, 121, 118, 118, 115, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139801764Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[0]R generation[0..12]: [58, 52, 56, 59, 60, 61, 61, 57, 56, 56, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139819864Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139837754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[1]R flush_numbers[0..12]: [122, 112, 118, 124, 125, 128, 128, 121, 118, 118, 115, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139855314Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[1]R generation[0..12]: [58, 52, 56, 59, 60, 61, 61, 57, 56, 56, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139872555Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139890385Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[2]R flush_numbers[0..12]: [122, 112, 118, 124, 125, 128, 128, 121, 118, 118, 115, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139907755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[2]R generation[0..12]: [58, 52, 56, 59, 60, 61, 61, 57, 56, 56, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139928805Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139947265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139963596Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.139985696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140002696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140021317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"":"downstairs","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140049807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140073957Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"0","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140098858Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"1","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140116108Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"client":"2","":"downstairs","session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140132768Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140148238Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"68d182d2-8f0f-4b52-8ece-ff3141b29258 is now active with session: f7fe2dae-7cd6-434f-b32e-5530f74278ee","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140164258Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140179308Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430,"session_id":"f7fe2dae-7cd6-434f-b32e-5530f74278ee"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:06.140194709Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 292 len 28672 data: 3 5 5 4 4 4 4 02/30 Read block 561 len 4096 03/30 Flush 04/30 Write block 130 len 20480 data: 9 7 6 5 6 05/30 Write block 274 len 4096 data: 4 06/30 Read block 115 len 28672 07/30 Write block 344 len 20480 data: 5 4 3 3 4 08/30 Flush 09/30 Write block 134 len 40960 data: 7 7 5 6 8 8 4 4 5 7 10/30 Read block 529 len 20480 11/30 Write block 548 len 32768 data: 6 9 11 9 8 7 7 6 12/30 Write block 81 len 32768 data: 3 3 4 4 5 5 6 6 13/30 Write block 31 len 4096 data: 5 14/30 Read block 374 len 8192 15/30 Read block 50 len 28672 16/30 Write block 2 len 28672 data: 7 8 7 8 6 5 5 17/30 Write block 316 len 36864 data: 4 5 5 6 6 7 8 7 6 18/30 Read block 214 len 20480 19/30 Write block 141 len 4096 data: 5 20/30 Read block 536 len 36864 21/30 Read block 273 len 16384 22/30 Read block 290 len 40960 23/30 Flush 24/30 Write block 332 len 4096 data: 3 25/30 Read block 216 len 4096 26/30 Read block 501 len 28672 27/30 Write block 491 len 28672 data: 5 4 5 6 5 6 6 28/30 Read block 458 len 24576 29/30 Write block 528 len 20480 data: 8 7 8 9 8 30/30 Read block 40 len 24576 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 5 Done Done Done false 31 Acked 1030 Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:39:06.168924333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 132 132 132 F F F 1 020-039 62 62 62 131 131 131 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 59 59 59 124 124 124 F F F 4 080-099 62 62 62 131 131 131 F F F 5 100-119 61 61 61 128 128 128 F F F 6 120-139 62 62 62 131 131 131 F F F 7 140-159 62 62 62 132 132 132 F F F 8 160-179 56 56 56 118 118 118 F F F 9 180-199 56 56 56 118 118 118 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 61 61 61 128 128 128 F F F 12 240-259 60 60 60 125 125 125 F F F 13 260-279 62 62 62 130 130 130 F F F 14 280-299 62 62 62 129 129 129 F F F 15 300-319 62 62 62 132 132 132 F F F 16 320-339 62 62 62 132 132 132 T T T 17 340-359 62 62 62 130 130 130 F F F 18 360-379 59 59 59 124 124 124 F F F 19 380-399 61 61 61 128 128 128 F F F 20 400-419 55 55 55 116 116 116 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 56 56 56 120 120 120 F F F 23 460-479 58 58 58 122 122 122 F F F 24 480-499 58 58 58 122 122 122 T T T 25 500-519 58 58 58 123 123 123 F F F 26 520-539 61 61 61 128 128 128 T T T 27 540-559 62 62 62 131 131 131 F F F 28 560-579 55 55 55 116 116 116 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 62, Max flush: 132 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 132 132 132 F F F 1 020-039 62 62 62 131 131 131 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 59 59 59 124 124 124 F F F 4 080-099 62 62 62 131 131 131 F F F 5 100-119 61 61 61 128 128 128 F F F 6 120-139 62 62 62 131 131 131 F F F 7 140-159 62 62 62 132 132 132 F F F 8 160-179 56 56 56 118 118 118 F F F 9 180-199 56 56 56 118 118 118 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 61 61 61 128 128 128 F F F 12 240-259 60 60 60 125 125 125 F F F 13 260-279 62 62 62 130 130 130 F F F 14 280-299 62 62 62 129 129 129 F F F 15 300-319 62 62 62 132 132 132 F F F 16 320-339 63 63 63 133 133 133 F F F 17 340-359 62 62 62 130 130 130 F F F 18 360-379 59 59 59 124 124 124 F F F 19 380-399 61 61 61 128 128 128 F F F 20 400-419 55 55 55 116 116 116 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 56 56 56 120 120 120 F F F 23 460-479 58 58 58 122 122 122 F F F 24 480-499 63 63 63 133 133 133 F F F 25 500-519 58 58 58 123 123 123 F F F 26 520-539 63 63 63 133 133 133 F F F 27 540-559 62 62 62 131 131 131 F F F 28 560-579 55 55 55 116 116 116 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 63, Max flush: 133 Begin loop 32 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.371725397Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.372027751Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.372058021Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438} {"msg":"Crucible d6e9e089-6bcd-4e0a-b0b9-5048014565ac has session id: 561aafe8-07b7-4ccd-8d72-0eb02079558b","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.372453806Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Upstairs opts: Upstairs UUID: d6e9e089-6bcd-4e0a-b0b9-5048014565ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.372499027Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Crucible stats registered with UUID: d6e9e089-6bcd-4e0a-b0b9-5048014565ac","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.372524917Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.372873402Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438} {"msg":"d6e9e089-6bcd-4e0a-b0b9-5048014565ac active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.372967463Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.372987423Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373002873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373020823Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373034624Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373048044Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373061614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373118745Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373142185Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373156865Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57626)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373461949Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50692)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373631961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:39:20.373757133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:39:20.373792363Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:39:20.373820414Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373843144Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373869604Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.373922115Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.374162158Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.374185898Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.374199769Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.374213469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.374798326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.374826586Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.375313913Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.375340293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.375355413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.375382004Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.375396584Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:20.375409994Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.375094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38286)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.375246202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.376219674Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.376263055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.376916873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.37749603Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377523171Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[0]R flush_numbers[0..12]: [132, 131, 118, 124, 131, 128, 131, 132, 118, 118, 115, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377539051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[0]R generation[0..12]: [62, 62, 56, 59, 62, 61, 62, 62, 56, 56, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377553581Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377568391Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[1]R flush_numbers[0..12]: [132, 131, 118, 124, 131, 128, 131, 132, 118, 118, 115, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377582721Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[1]R generation[0..12]: [62, 62, 56, 59, 62, 61, 62, 62, 56, 56, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377596692Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377610942Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[2]R flush_numbers[0..12]: [132, 131, 118, 124, 131, 128, 131, 132, 118, 118, 115, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377624912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[2]R generation[0..12]: [62, 62, 56, 59, 62, 61, 62, 62, 56, 56, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377638672Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377652652Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Max found gen is 64Wait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377669493Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377687973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377701613Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377714713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"":"downstairs","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377728133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377740014Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"0","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377753374Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"1","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377766614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"client":"2","":"downstairs","session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377779904Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377792034Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"d6e9e089-6bcd-4e0a-b0b9-5048014565ac is now active with session: 561aafe8-07b7-4ccd-8d72-0eb02079558b","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377804884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377816775Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438,"session_id":"561aafe8-07b7-4ccd-8d72-0eb02079558b"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:30.377829024Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 352 len 36864 data: 5 6 8 8 8 7 7 7 6 02/30 Read block 451 len 40960 03/30 Read block 83 len 28672 04/30 Write block 386 len 36864 data: 5 3 4 4 4 4 4 4 4 05/30 Read block 492 len 32768 06/30 Read block 463 len 28672 07/30 Write block 370 len 16384 data: 10 9 7 8 08/30 Write block 118 len 20480 data: 5 5 6 5 4 09/30 Write block 101 len 16384 data: 4 3 4 4 10/30 Read block 112 len 28672 11/30 Read block 163 len 24576 12/30 Write block 562 len 28672 data: 4 3 3 3 3 5 5 13/30 Write block 475 len 8192 data: 5 7 14/30 Read block 582 len 28672 15/30 Write block 503 len 20480 data: 9 9 7 7 5 16/30 Write block 324 len 36864 data: 7 7 6 6 5 5 5 4 4 17/30 Read block 508 len 16384 18/30 Read block 297 len 12288 19/30 Write block 347 len 24576 data: 4 5 3 4 5 6 20/30 Write block 475 len 20480 data: 6 8 7 7 5 21/30 Read block 483 len 24576 22/30 Write block 179 len 24576 data: 5 4 3 4 3 3 23/30 Write block 270 len 4096 data: 3 24/30 Write block 390 len 28672 data: 5 5 5 5 5 5 6 25/30 Flush 26/30 Write block 341 len 40960 data: 7 7 6 6 5 4 5 6 4 5 27/30 Write block 141 len 40960 data: 6 6 8 7 6 5 7 8 8 5 28/30 Write block 403 len 16384 data: 4 3 4 4 29/30 Write block 100 len 16384 data: 6 5 4 5 30/30 Read block 445 len 16384 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Write 4 Done Done Done false 31 Acked 1030 Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:39:30.404720266Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 132 132 132 F F F 1 020-039 62 62 62 131 131 131 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 59 59 59 124 124 124 F F F 4 080-099 62 62 62 131 131 131 F F F 5 100-119 64 64 64 134 134 134 T T T 6 120-139 64 64 64 134 134 134 F F F 7 140-159 62 62 62 132 132 132 T T T 8 160-179 64 64 64 135 135 135 F F F 9 180-199 64 64 64 135 135 135 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 61 61 61 128 128 128 F F F 12 240-259 60 60 60 125 125 125 F F F 13 260-279 64 64 64 135 135 135 F F F 14 280-299 62 62 62 129 129 129 F F F 15 300-319 62 62 62 132 132 132 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 64 64 64 134 134 134 T T T 18 360-379 64 64 64 134 134 134 F F F 19 380-399 64 64 64 135 135 135 F F F 20 400-419 55 55 55 116 116 116 T T T 21 420-439 57 57 57 121 121 121 F F F 22 440-459 56 56 56 120 120 120 F F F 23 460-479 64 64 64 134 134 134 F F F 24 480-499 63 63 63 133 133 133 F F F 25 500-519 64 64 64 134 134 134 F F F 26 520-539 63 63 63 133 133 133 F F F 27 540-559 62 62 62 131 131 131 F F F 28 560-579 64 64 64 134 134 134 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 64, Max flush: 135 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 132 132 132 F F F 1 020-039 62 62 62 131 131 131 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 59 59 59 124 124 124 F F F 4 080-099 62 62 62 131 131 131 F F F 5 100-119 65 65 65 136 136 136 F F F 6 120-139 64 64 64 134 134 134 F F F 7 140-159 65 65 65 136 136 136 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 64 64 64 135 135 135 F F F 10 200-219 54 54 54 115 115 115 F F F 11 220-239 61 61 61 128 128 128 F F F 12 240-259 60 60 60 125 125 125 F F F 13 260-279 64 64 64 135 135 135 F F F 14 280-299 62 62 62 129 129 129 F F F 15 300-319 62 62 62 132 132 132 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 65 65 65 136 136 136 F F F 18 360-379 64 64 64 134 134 134 F F F 19 380-399 64 64 64 135 135 135 F F F 20 400-419 65 65 65 136 136 136 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 56 56 56 120 120 120 F F F 23 460-479 64 64 64 134 134 134 F F F 24 480-499 63 63 63 133 133 133 F F F 25 500-519 64 64 64 134 134 134 F F F 26 520-539 63 63 63 133 133 133 F F F 27 540-559 62 62 62 131 131 131 F F F 28 560-579 64 64 64 134 134 134 F F F 29 580-599 50 50 50 108 108 108 F F F Max gen: 65, Max flush: 136 Begin loop 33 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.619592212Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.619945137Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.619969977Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446} {"msg":"Crucible ced41cc9-54ff-4f37-9a20-420cff8d87d4 has session id: 1859c623-09cc-4aae-b0f8-2760a9ce19a8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620380462Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Upstairs opts: Upstairs UUID: ced41cc9-54ff-4f37-9a20-420cff8d87d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620428433Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Crucible stats registered with UUID: ced41cc9-54ff-4f37-9a20-420cff8d87d4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620449163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620732407Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446} {"msg":"ced41cc9-54ff-4f37-9a20-420cff8d87d4 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620761117Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620775698Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620794248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620808278Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620822138Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620835638Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620849399Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620915819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.620935349Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.62094949Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:39:44.621128052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:53414)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.621227163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:39:44.621262664Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:39:44.621287574Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.621309214Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.621331685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62890)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.621354345Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.621514167Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.621925352Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.621948482Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.621962373Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.621975813Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.622490349Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.622612661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.622963755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.622998216Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.623020776Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.623074957Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.623101117Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:44.623115417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.622988406Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52851)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.623230229Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.624407593Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.624446344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.62490897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625432146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625457837Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[0]R flush_numbers[0..12]: [132, 131, 118, 124, 131, 136, 134, 136, 135, 135, 115, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625477487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[0]R generation[0..12]: [62, 62, 56, 59, 62, 65, 64, 65, 64, 64, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625495267Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625513087Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[1]R flush_numbers[0..12]: [132, 131, 118, 124, 131, 136, 134, 136, 135, 135, 115, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625530808Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[1]R generation[0..12]: [62, 62, 56, 59, 62, 65, 64, 65, 64, 64, 54, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625548168Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625566178Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[2]R flush_numbers[0..12]: [132, 131, 118, 124, 131, 136, 134, 136, 135, 135, 115, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625583728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[2]R generation[0..12]: [62, 62, 56, 59, 62, 65, 64, 65, 64, 64, 54, 61]","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2024-07-10T15:39:54.625604139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625622099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625643789Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625661229Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.62567773Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.62569422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"":"downstairs","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.62571079Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.62572685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"0","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625740501Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"1","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625753981Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"client":"2","":"downstairs","session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625767281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625779781Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"ced41cc9-54ff-4f37-9a20-420cff8d87d4 is now active with session: 1859c623-09cc-4aae-b0f8-2760a9ce19a8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625792661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625804491Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446,"session_id":"1859c623-09cc-4aae-b0f8-2760a9ce19a8"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:39:54.625816822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 192 len 20480 data: 7 6 6 6 6 02/30 Write block 2 len 12288 data: 8 9 8 03/30 Write block 252 len 40960 data: 10 9 9 10 7 7 7 7 8 6 04/30 Write block 486 len 16384 data: 4 4 4 4 05/30 Write block 591 len 32768 data: 8 7 4 5 4 3 3 3 06/30 Read block 32 len 40960 07/30 Read block 192 len 20480 08/30 Write block 447 len 32768 data: 7 6 4 5 6 7 7 5 09/30 Read block 344 len 4096 10/30 Read block 127 len 28672 11/30 Write block 570 len 8192 data: 4 4 12/30 Write block 137 len 32768 data: 7 9 9 5 7 7 9 8 13/30 Read block 566 len 4096 14/30 Write block 25 len 12288 data: 5 3 3 15/30 Write block 401 len 28672 data: 5 4 5 4 5 5 5 16/30 Read block 57 len 16384 17/30 Read block 516 len 8192 18/30 Write block 564 len 12288 data: 4 4 4 19/30 Write block 69 len 12288 data: 6 6 4 20/30 Read block 133 len 4096 21/30 Read block 207 len 16384 22/30 Read block 569 len 16384 23/30 Write block 208 len 36864 data: 5 5 6 7 6 5 4 4 4 24/30 Read block 30 len 28672 25/30 Read block 224 len 24576 26/30 Write block 586 len 24576 data: 4 4 5 5 7 9 27/30 Write block 471 len 32768 data: 3 2 5 5 7 9 8 8 28/30 Read block 49 len 16384 29/30 Write block 65 len 24576 data: 8 7 7 7 7 7 30/30 Read block 397 len 36864 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 6 Done Done Done false 31 Acked 1030 Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:39:54.653484833Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 137 137 137 F F F 1 020-039 66 66 66 137 137 137 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 66 66 66 137 137 137 T T T 4 080-099 62 62 62 131 131 131 F F F 5 100-119 65 65 65 136 136 136 F F F 6 120-139 66 66 66 137 137 137 F F F 7 140-159 66 66 66 137 137 137 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 66 66 66 137 137 137 F F F 11 220-239 61 61 61 128 128 128 F F F 12 240-259 66 66 66 137 137 137 F F F 13 260-279 66 66 66 137 137 137 F F F 14 280-299 62 62 62 129 129 129 F F F 15 300-319 62 62 62 132 132 132 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 65 65 65 136 136 136 F F F 18 360-379 64 64 64 134 134 134 F F F 19 380-399 64 64 64 135 135 135 F F F 20 400-419 66 66 66 137 137 137 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 66 66 66 137 137 137 F F F 23 460-479 64 64 64 134 134 134 T T T 24 480-499 66 66 66 137 137 137 F F F 25 500-519 64 64 64 134 134 134 F F F 26 520-539 63 63 63 133 133 133 F F F 27 540-559 62 62 62 131 131 131 F F F 28 560-579 66 66 66 137 137 137 F F F 29 580-599 66 66 66 137 137 137 T T T Max gen: 66, Max flush: 137 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 137 137 137 F F F 1 020-039 66 66 66 137 137 137 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 67 67 67 138 138 138 F F F 4 080-099 62 62 62 131 131 131 F F F 5 100-119 65 65 65 136 136 136 F F F 6 120-139 66 66 66 137 137 137 F F F 7 140-159 66 66 66 137 137 137 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 66 66 66 137 137 137 F F F 11 220-239 61 61 61 128 128 128 F F F 12 240-259 66 66 66 137 137 137 F F F 13 260-279 66 66 66 137 137 137 F F F 14 280-299 62 62 62 129 129 129 F F F 15 300-319 62 62 62 132 132 132 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 65 65 65 136 136 136 F F F 18 360-379 64 64 64 134 134 134 F F F 19 380-399 64 64 64 135 135 135 F F F 20 400-419 66 66 66 137 137 137 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 66 66 66 137 137 137 F F F 23 460-479 67 67 67 138 138 138 F F F 24 480-499 66 66 66 137 137 137 F F F 25 500-519 64 64 64 134 134 134 F F F 26 520-539 63 63 63 133 133 133 F F F 27 540-559 62 62 62 131 131 131 F F F 28 560-579 66 66 66 137 137 137 F F F 29 580-599 67 67 67 138 138 138 F F F Max gen: 67, Max flush: 138 Begin loop 34 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.853389319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.853934036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.853965187Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454} {"msg":"Crucible 77d14e24-ce28-4710-adb1-419de215ecd6 has session id: ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.85420276Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Upstairs opts: Upstairs UUID: 77d14e24-ce28-4710-adb1-419de215ecd6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854301401Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Crucible stats registered with UUID: 77d14e24-ce28-4710-adb1-419de215ecd6","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854331871Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854710636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454} {"msg":"77d14e24-ce28-4710-adb1-419de215ecd6 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854741096Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854769937Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854793897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854816528Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854840018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854863098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854888418Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854915479Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854942559Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.854970479Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:08.85500177Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:08.85503114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_connection connected from Ok(127.0.0.1:43304)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.855057391Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:08.855085321Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.855139052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.855198042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_connection connected from Ok(127.0.0.1:36719)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.855231863Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.855426235Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.855973792Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.856011072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.856040163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.856066393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.856679531Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.856706752Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.857260699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.857288329Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.857304269Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.857316989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.857331289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:08.85734492Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.857047116Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_connection connected from Ok(127.0.0.1:61200)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.857350869Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.858227171Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.858259051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.858751537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859304624Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859328525Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[0]R flush_numbers[0..12]: [137, 137, 118, 138, 131, 136, 137, 137, 135, 137, 137, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859347955Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[0]R generation[0..12]: [66, 66, 56, 67, 62, 65, 66, 66, 64, 66, 66, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859365685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859383625Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[1]R flush_numbers[0..12]: [137, 137, 118, 138, 131, 136, 137, 137, 135, 137, 137, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859401335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[1]R generation[0..12]: [66, 66, 56, 67, 62, 65, 66, 66, 64, 66, 66, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859418466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859436196Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[2]R flush_numbers[0..12]: [137, 137, 118, 138, 131, 136, 137, 137, 135, 137, 137, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859453836Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[2]R generation[0..12]: [66, 66, 56, 67, 62, 65, 66, 66, 64, 66, 66, 61]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-07-10T15:40:18.859470806Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859498417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859515237Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859532187Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859548507Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859565008Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"":"downstairs","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859581858Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859596948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"0","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859613798Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"1","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859630488Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"client":"2","":"downstairs","session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859647149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859662569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"77d14e24-ce28-4710-adb1-419de215ecd6 is now active with session: ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859678909Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859694139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "pid":1454,"session_id":"ab9fbf9d-9745-48b7-82ab-bb9a91cb0d48"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:18.859714539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 125 len 4096 data: 3 03/30 Read block 368 len 32768 04/30 Read block 106 len 8192 05/30 Read block 418 len 16384 06/30 Flush 07/30 Flush 08/30 Write block 393 len 32768 data: 6 6 6 7 6 6 6 6 09/30 Write block 501 len 24576 data: 9 9 10 10 8 8 10/30 Read block 289 len 28672 11/30 Read block 503 len 12288 12/30 Read block 243 len 32768 13/30 Read block 575 len 4096 14/30 Read block 109 len 8192 15/30 Read block 516 len 32768 16/30 Read block 175 len 16384 17/30 Flush 18/30 Write block 517 len 12288 data: 2 3 2 19/30 Read block 310 len 16384 20/30 Write block 538 len 36864 data: 9 7 6 4 6 7 9 5 5 21/30 Read block 97 len 16384 22/30 Write block 576 len 4096 data: 5 23/30 Read block 376 len 4096 24/30 Write block 399 len 36864 data: 7 7 6 5 6 5 6 6 6 25/30 Read block 564 len 24576 26/30 Read block 380 len 12288 27/30 Write block 250 len 12288 data: 11 12 11 28/30 Read block 292 len 20480 29/30 Write block 241 len 16384 data: 4 3 4 6 30/30 Write block 376 len 8192 data: 8 11 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Read 4 Done Done Done false 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Write 4 Done Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 2 2 5 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:18.884539745Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 137 137 137 F F F 1 020-039 66 66 66 137 137 137 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 67 67 67 138 138 138 F F F 4 080-099 62 62 62 131 131 131 F F F 5 100-119 65 65 65 136 136 136 F F F 6 120-139 68 68 68 140 140 140 F F F 7 140-159 66 66 66 137 137 137 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 66 66 66 137 137 137 F F F 11 220-239 61 61 61 128 128 128 F F F 12 240-259 66 66 66 137 137 137 T T T 13 260-279 66 66 66 137 137 137 F F F 14 280-299 62 62 62 129 129 129 F F F 15 300-319 62 62 62 132 132 132 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 65 65 65 136 136 136 F F F 18 360-379 64 64 64 134 134 134 T T T 19 380-399 68 68 68 142 142 142 T T T 20 400-419 68 68 68 142 142 142 T T T 21 420-439 57 57 57 121 121 121 F F F 22 440-459 66 66 66 137 137 137 F F F 23 460-479 67 67 67 138 138 138 F F F 24 480-499 66 66 66 137 137 137 F F F 25 500-519 68 68 68 142 142 142 T T T 26 520-539 63 63 63 133 133 133 T T T 27 540-559 62 62 62 131 131 131 T T T 28 560-579 66 66 66 137 137 137 T T T 29 580-599 67 67 67 138 138 138 F F F Max gen: 68, Max flush: 142 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 137 137 137 F F F 1 020-039 66 66 66 137 137 137 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 67 67 67 138 138 138 F F F 4 080-099 62 62 62 131 131 131 F F F 5 100-119 65 65 65 136 136 136 F F F 6 120-139 68 68 68 140 140 140 F F F 7 140-159 66 66 66 137 137 137 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 66 66 66 137 137 137 F F F 11 220-239 61 61 61 128 128 128 F F F 12 240-259 69 69 69 143 143 143 F F F 13 260-279 66 66 66 137 137 137 F F F 14 280-299 62 62 62 129 129 129 F F F 15 300-319 62 62 62 132 132 132 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 65 65 65 136 136 136 F F F 18 360-379 69 69 69 143 143 143 F F F 19 380-399 69 69 69 143 143 143 F F F 20 400-419 69 69 69 143 143 143 F F F 21 420-439 57 57 57 121 121 121 F F F 22 440-459 66 66 66 137 137 137 F F F 23 460-479 67 67 67 138 138 138 F F F 24 480-499 66 66 66 137 137 137 F F F 25 500-519 69 69 69 143 143 143 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 69 69 69 143 143 143 F F F 28 560-579 69 69 69 143 143 143 F F F 29 580-599 67 67 67 138 138 138 F F F Max gen: 69, Max flush: 143 Begin loop 35 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.150677372Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151030726Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151059867Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462} {"msg":"Crucible 04871a66-2c85-449d-ac25-a3978027ec43 has session id: 4b27cab2-b097-4fdd-8f56-86392faf7db3","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151467032Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Upstairs opts: Upstairs UUID: 04871a66-2c85-449d-ac25-a3978027ec43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151499192Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Crucible stats registered with UUID: 04871a66-2c85-449d-ac25-a3978027ec43","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151514142Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151784686Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462} {"msg":"04871a66-2c85-449d-ac25-a3978027ec43 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151888607Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151910638Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151936108Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151960718Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.151982419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.152004469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.152029369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.152052789Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.15207739Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.15210139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_connection connected from Ok(127.0.0.1:32869)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.152357833Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_connection connected from Ok(127.0.0.1:65024)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.152563786Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:33.152617087Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:33.152650907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:33.152669797Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.152686417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.152700297Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.152843639Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.153053522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.153074132Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.153088082Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.153101713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.153635109Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.153740111Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.154111426Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.154149086Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.154177076Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.154216357Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.154245657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:33.154262517Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.154755563Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40717)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.154951076Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.155937118Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.155962259Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.156489296Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157043972Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157072283Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[0]R flush_numbers[0..12]: [137, 137, 118, 138, 131, 136, 140, 137, 135, 137, 137, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157090893Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[0]R generation[0..12]: [66, 66, 56, 67, 62, 65, 68, 66, 64, 66, 66, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157108553Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157126264Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[1]R flush_numbers[0..12]: [137, 137, 118, 138, 131, 136, 140, 137, 135, 137, 137, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157143884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[1]R generation[0..12]: [66, 66, 56, 67, 62, 65, 68, 66, 64, 66, 66, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157161614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157179354Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[2]R flush_numbers[0..12]: [137, 137, 118, 138, 131, 136, 140, 137, 135, 137, 137, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157196905Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[2]R generation[0..12]: [66, 66, 56, 67, 62, 65, 68, 66, 64, 66, 66, 61]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157223045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157241465Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157257775Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157274566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157290946Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157307306Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"":"downstairs","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157325756Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157351336Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"0","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157381087Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"1","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2024-07-10T15:40:43.157410857Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"client":"2","":"downstairs","session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157444968Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157469318Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"04871a66-2c85-449d-ac25-a3978027ec43 is now active with session: 4b27cab2-b097-4fdd-8f56-86392faf7db3","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157486608Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157501888Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462,"session_id":"4b27cab2-b097-4fdd-8f56-86392faf7db3"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:43.157517298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 435 len 16384 data: 4 6 5 5 02/30 Write block 353 len 36864 data: 7 9 9 9 8 8 8 7 7 03/30 Write block 551 len 32768 data: 10 9 8 8 7 4 7 5 04/30 Read block 328 len 8192 05/30 Write block 296 len 20480 data: 5 5 5 4 4 06/30 Read block 481 len 16384 07/30 Read block 33 len 16384 08/30 Write block 291 len 8192 data: 3 4 09/30 Write block 437 len 24576 data: 6 6 7 7 11 9 10/30 Write block 406 len 16384 data: 7 7 5 5 11/30 Read block 128 len 40960 12/30 Read block 554 len 28672 13/30 Read block 196 len 40960 14/30 Read block 142 len 40960 15/30 Read block 494 len 36864 16/30 Write block 87 len 28672 data: 7 7 7 7 6 5 5 17/30 Write block 236 len 8192 data: 9 7 18/30 Flush 19/30 Write block 113 len 40960 data: 5 5 6 6 5 6 6 7 6 5 20/30 Read block 33 len 12288 21/30 Read block 248 len 20480 22/30 Write block 71 len 40960 data: 5 4 5 5 5 4 4 4 4 3 23/30 Write block 574 len 16384 data: 4 4 6 5 24/30 Write block 146 len 28672 data: 6 8 9 9 6 5 5 25/30 Write block 482 len 12288 data: 5 5 4 26/30 Write block 6 len 16384 data: 7 6 6 5 27/30 Read block 538 len 4096 28/30 Read block 122 len 20480 29/30 Write block 203 len 32768 data: 5 5 4 4 5 6 6 7 30/30 Read block 557 len 36864 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 10 Done Done Done false 20 Acked 1019 Read 3 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Write 8 Done Done Done false 30 Acked 1029 Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 12 11 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:43.183192614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 137 137 137 T T T 1 020-039 66 66 66 137 137 137 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 67 67 67 138 138 138 T T T 4 080-099 70 70 70 144 144 144 T T T 5 100-119 65 65 65 136 136 136 T T T 6 120-139 68 68 68 140 140 140 T T T 7 140-159 66 66 66 137 137 137 T T T 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 66 66 66 137 137 137 T T T 11 220-239 70 70 70 144 144 144 F F F 12 240-259 69 69 69 143 143 143 F F F 13 260-279 66 66 66 137 137 137 F F F 14 280-299 70 70 70 144 144 144 F F F 15 300-319 70 70 70 144 144 144 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 70 70 70 144 144 144 F F F 18 360-379 70 70 70 144 144 144 F F F 19 380-399 69 69 69 143 143 143 F F F 20 400-419 70 70 70 144 144 144 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 67 67 67 138 138 138 F F F 24 480-499 66 66 66 137 137 137 T T T 25 500-519 69 69 69 143 143 143 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 70 70 70 144 144 144 F F F 28 560-579 69 69 69 143 143 143 T T T 29 580-599 67 67 67 138 138 138 F F F Max gen: 70, Max flush: 144 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 145 145 145 F F F 1 020-039 66 66 66 137 137 137 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 71 71 71 145 145 145 F F F 4 080-099 71 71 71 145 145 145 F F F 5 100-119 71 71 71 145 145 145 F F F 6 120-139 71 71 71 145 145 145 F F F 7 140-159 71 71 71 145 145 145 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 71 71 71 145 145 145 F F F 11 220-239 70 70 70 144 144 144 F F F 12 240-259 69 69 69 143 143 143 F F F 13 260-279 66 66 66 137 137 137 F F F 14 280-299 70 70 70 144 144 144 F F F 15 300-319 70 70 70 144 144 144 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 70 70 70 144 144 144 F F F 18 360-379 70 70 70 144 144 144 F F F 19 380-399 69 69 69 143 143 143 F F F 20 400-419 70 70 70 144 144 144 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 67 67 67 138 138 138 F F F 24 480-499 71 71 71 145 145 145 F F F 25 500-519 69 69 69 143 143 143 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 70 70 70 144 144 144 F F F 28 560-579 71 71 71 145 145 145 F F F 29 580-599 67 67 67 138 138 138 F F F Max gen: 71, Max flush: 145 Begin loop 36 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.462146184Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.462470749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.462495689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470} {"msg":"Crucible dc5558c8-af2e-461f-bfe6-38f8815e5886 has session id: 8006d5d1-a708-4b3c-bcf9-e2c5605dfda8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.462903324Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Upstairs opts: Upstairs UUID: dc5558c8-af2e-461f-bfe6-38f8815e5886, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.462951445Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Crucible stats registered with UUID: dc5558c8-af2e-461f-bfe6-38f8815e5886","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.462978345Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463272639Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470} {"msg":"dc5558c8-af2e-461f-bfe6-38f8815e5886 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.46336379Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.46338286Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463397931Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463411661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463425241Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463438631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463452121Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463508772Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463536982Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463552953Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41861)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.463802716Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_connection connected from Ok(127.0.0.1:57298)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.464059329Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:57.464192461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:57.464225381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:40:57.464243891Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.464265351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.464290872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.464317192Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.464647046Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.464666696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.464680297Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.464693887Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.465297044Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.465323715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.465849211Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.465874922Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.465889262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.465901742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.465920203Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:40:57.465933943Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.465121732Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43923)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.465281234Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.466843694Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.466876084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.467421701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.467958678Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.467987598Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[0]R flush_numbers[0..12]: [145, 137, 118, 145, 145, 145, 145, 145, 135, 137, 145, 144]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468006449Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[0]R generation[0..12]: [71, 66, 56, 71, 71, 71, 71, 71, 64, 66, 71, 70]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468024559Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468042539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[1]R flush_numbers[0..12]: [145, 137, 118, 145, 145, 145, 145, 145, 135, 137, 145, 144]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.46806023Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[1]R generation[0..12]: [71, 66, 56, 71, 71, 71, 71, 71, 64, 66, 71, 70]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.46807772Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.46809551Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[2]R flush_numbers[0..12]: [145, 137, 118, 145, 145, 145, 145, 145, 135, 137, 145, 144]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.46811815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"[2]R generation[0..12]: [71, 66, 56, 71, 71, 71, 71, 71, 64, 66, 71, 70]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.46813566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468156341Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468173221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468190071Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468206611Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468222952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"":"downstairs","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468239932Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468255042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"0","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468271742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"1","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468288482Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"client":"2","":"downstairs","session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468305473Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468320883Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"dc5558c8-af2e-461f-bfe6-38f8815e5886 is now active with session: 8006d5d1-a708-4b3c-bcf9-e2c5605dfda8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468337003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468352293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470,"session_id":"8006d5d1-a708-4b3c-bcf9-e2c5605dfda8"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:07.468370983Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 478 len 16384 data: 9 6 4 5 02/30 Read block 107 len 36864 03/30 Read block 271 len 28672 04/30 Read block 164 len 40960 05/30 Write block 351 len 32768 data: 6 7 8 10 10 10 9 9 06/30 Read block 85 len 12288 07/30 Read block 42 len 20480 08/30 Write block 221 len 36864 data: 6 7 6 8 7 7 8 8 9 09/30 Read block 555 len 28672 10/30 Read block 127 len 28672 11/30 Flush 12/30 Write block 239 len 24576 data: 6 4 5 4 5 7 13/30 Read block 437 len 8192 14/30 Write block 509 len 16384 data: 6 5 5 3 15/30 Read block 140 len 16384 16/30 Write block 112 len 24576 data: 5 6 6 7 7 6 17/30 Read block 21 len 8192 18/30 Write block 482 len 16384 data: 6 6 5 5 19/30 Write block 219 len 40960 data: 4 4 7 8 7 9 8 8 9 9 20/30 Flush 21/30 Write block 480 len 40960 data: 5 6 7 7 6 6 5 5 5 5 22/30 Read block 3 len 32768 23/30 Read block 331 len 8192 24/30 Read block 407 len 8192 25/30 Flush 26/30 Write block 278 len 4096 data: 6 27/30 Read block 484 len 32768 28/30 Read block 322 len 20480 29/30 Read block 459 len 4096 30/30 Read block 129 len 36864 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:41:07.496843845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 145 145 145 F F F 1 020-039 66 66 66 137 137 137 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 71 71 71 145 145 145 F F F 4 080-099 71 71 71 145 145 145 F F F 5 100-119 72 72 72 147 147 147 F F F 6 120-139 71 71 71 145 145 145 F F F 7 140-159 71 71 71 145 145 145 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 72 72 72 147 147 147 F F F 12 240-259 72 72 72 147 147 147 F F F 13 260-279 66 66 66 137 137 137 T T T 14 280-299 70 70 70 144 144 144 F F F 15 300-319 70 70 70 144 144 144 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 72 72 72 146 146 146 F F F 18 360-379 70 70 70 144 144 144 F F F 19 380-399 69 69 69 143 143 143 F F F 20 400-419 70 70 70 144 144 144 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 72 72 72 146 146 146 F F F 24 480-499 72 72 72 148 148 148 F F F 25 500-519 72 72 72 147 147 147 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 70 70 70 144 144 144 F F F 28 560-579 71 71 71 145 145 145 F F F 29 580-599 67 67 67 138 138 138 F F F Max gen: 72, Max flush: 148 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 145 145 145 F F F 1 020-039 66 66 66 137 137 137 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 71 71 71 145 145 145 F F F 4 080-099 71 71 71 145 145 145 F F F 5 100-119 72 72 72 147 147 147 F F F 6 120-139 71 71 71 145 145 145 F F F 7 140-159 71 71 71 145 145 145 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 72 72 72 147 147 147 F F F 12 240-259 72 72 72 147 147 147 F F F 13 260-279 73 73 73 149 149 149 F F F 14 280-299 70 70 70 144 144 144 F F F 15 300-319 70 70 70 144 144 144 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 72 72 72 146 146 146 F F F 18 360-379 70 70 70 144 144 144 F F F 19 380-399 69 69 69 143 143 143 F F F 20 400-419 70 70 70 144 144 144 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 72 72 72 146 146 146 F F F 24 480-499 72 72 72 148 148 148 F F F 25 500-519 72 72 72 147 147 147 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 70 70 70 144 144 144 F F F 28 560-579 71 71 71 145 145 145 F F F 29 580-599 67 67 67 138 138 138 F F F Max gen: 73, Max flush: 149 Begin loop 37 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.675418551Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.675705485Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.675721825Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478} {"msg":"Crucible 8442920c-8542-4bfc-aa88-0142adecb4cf has session id: f0993650-b1a1-4057-a31d-2f238f98ac52","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676205411Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Upstairs opts: Upstairs UUID: 8442920c-8542-4bfc-aa88-0142adecb4cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676249892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Crucible stats registered with UUID: 8442920c-8542-4bfc-aa88-0142adecb4cf","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676273952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676660267Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478} {"msg":"8442920c-8542-4bfc-aa88-0142adecb4cf active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676694987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676708807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676723218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676737128Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676756048Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676779768Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676805619Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676831099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676854579Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.676871499Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:41:21.677220364Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_connection connected from Ok(127.0.0.1:53469)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.677258054Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:41:21.677286915Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:41:21.677319845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.677343055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.677367726Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_connection connected from Ok(127.0.0.1:54826)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.677391906Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.677417576Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.677960573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.677993303Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.678019864Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.678046344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.678665902Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.678693632Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.679197109Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.679220879Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.679235079Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.67929438Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.67931311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:21.6793299Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.677863952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_connection connected from Ok(127.0.0.1:43915)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.678038714Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.679189649Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.679218489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.679810347Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680374474Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680398704Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[0]R flush_numbers[0..12]: [145, 137, 118, 145, 145, 147, 145, 145, 135, 137, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680417544Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[0]R generation[0..12]: [71, 66, 56, 71, 71, 72, 71, 71, 64, 66, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680435374Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680453025Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[1]R flush_numbers[0..12]: [145, 137, 118, 145, 145, 147, 145, 145, 135, 137, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680470635Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[1]R generation[0..12]: [71, 66, 56, 71, 71, 72, 71, 71, 64, 66, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680487715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680505386Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[2]R flush_numbers[0..12]: [145, 137, 118, 145, 145, 147, 145, 145, 135, 137, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680528816Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[2]R generation[0..12]: [71, 66, 56, 71, 71, 72, 71, 71, 64, 66, 72, 72]","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680550396Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680568886Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680585416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680602137Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680618397Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680634787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"":"downstairs","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680651587Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680666668Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"0","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680683828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"1","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680700808Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"client":"2","":"downstairs","session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680717938Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680733398Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"8442920c-8542-4bfc-aa88-0142adecb4cf is now active with session: f0993650-b1a1-4057-a31d-2f238f98ac52","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680749409Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680764289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478,"session_id":"f0993650-b1a1-4057-a31d-2f238f98ac52"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:31.680781959Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 15 len 24576 data: 3 4 5 7 7 6 02/30 Write block 516 len 4096 data: 2 03/30 Flush 04/30 Write block 392 len 12288 data: 6 7 7 05/30 Read block 475 len 8192 06/30 Read block 138 len 40960 07/30 Write block 462 len 16384 data: 3 4 4 3 08/30 Write block 397 len 32768 data: 7 7 8 8 7 6 7 6 09/30 Read block 529 len 16384 10/30 Read block 278 len 40960 11/30 Read block 344 len 24576 12/30 Read block 398 len 28672 13/30 Read block 5 len 4096 14/30 Read block 221 len 32768 15/30 Write block 14 len 12288 data: 5 4 5 16/30 Write block 368 len 32768 data: 5 7 11 10 8 9 9 6 17/30 Read block 366 len 24576 18/30 Read block 522 len 32768 19/30 Write block 479 len 24576 data: 7 6 7 8 8 7 20/30 Write block 226 len 40960 data: 9 10 10 10 7 6 5 7 10 10 21/30 Read block 325 len 36864 22/30 Read block 298 len 20480 23/30 Read block 307 len 24576 24/30 Write block 105 len 20480 data: 3 3 2 4 4 25/30 Read block 403 len 28672 26/30 Read block 176 len 24576 27/30 Read block 18 len 12288 28/30 Read block 305 len 16384 29/30 Write block 288 len 28672 data: 4 4 3 4 5 6 6 30/30 Read block 310 len 28672 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 7 Done Done Done false 31 Acked 1030 Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:41:31.708019755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 151 151 151 F F F 1 020-039 74 74 74 150 150 150 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 71 71 71 145 145 145 F F F 4 080-099 71 71 71 145 145 145 F F F 5 100-119 72 72 72 147 147 147 T T T 6 120-139 71 71 71 145 145 145 F F F 7 140-159 71 71 71 145 145 145 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 74 74 74 151 151 151 F F F 12 240-259 72 72 72 147 147 147 F F F 13 260-279 73 73 73 149 149 149 F F F 14 280-299 70 70 70 144 144 144 T T T 15 300-319 70 70 70 144 144 144 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 72 72 72 146 146 146 F F F 18 360-379 74 74 74 151 151 151 F F F 19 380-399 74 74 74 151 151 151 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 74 74 74 151 151 151 F F F 24 480-499 74 74 74 151 151 151 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 70 70 70 144 144 144 F F F 28 560-579 71 71 71 145 145 145 F F F 29 580-599 67 67 67 138 138 138 F F F Max gen: 74, Max flush: 151 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 151 151 151 F F F 1 020-039 74 74 74 150 150 150 F F F 2 040-059 56 56 56 118 118 118 F F F 3 060-079 71 71 71 145 145 145 F F F 4 080-099 71 71 71 145 145 145 F F F 5 100-119 75 75 75 152 152 152 F F F 6 120-139 71 71 71 145 145 145 F F F 7 140-159 71 71 71 145 145 145 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 66 66 66 137 137 137 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 74 74 74 151 151 151 F F F 12 240-259 72 72 72 147 147 147 F F F 13 260-279 73 73 73 149 149 149 F F F 14 280-299 75 75 75 152 152 152 F F F 15 300-319 70 70 70 144 144 144 F F F 16 320-339 64 64 64 134 134 134 F F F 17 340-359 72 72 72 146 146 146 F F F 18 360-379 74 74 74 151 151 151 F F F 19 380-399 74 74 74 151 151 151 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 74 74 74 151 151 151 F F F 24 480-499 74 74 74 151 151 151 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 70 70 70 144 144 144 F F F 28 560-579 71 71 71 145 145 145 F F F 29 580-599 67 67 67 138 138 138 F F F Max gen: 75, Max flush: 152 Begin loop 38 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.900838002Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.901134575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.901160486Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486} {"msg":"Crucible 3dfd1ea8-f0d2-414f-b5ee-1eca50b653d2 has session id: 88ae22d3-fe9f-4f6b-b621-23599c87f6c9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.901580911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Upstairs opts: Upstairs UUID: 3dfd1ea8-f0d2-414f-b5ee-1eca50b653d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.901628392Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Crucible stats registered with UUID: 3dfd1ea8-f0d2-414f-b5ee-1eca50b653d2","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.901660292Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.901925575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486} {"msg":"3dfd1ea8-f0d2-414f-b5ee-1eca50b653d2 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902028277Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902048957Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902064217Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902078057Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902091647Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902109318Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902133968Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902150978Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902165448Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902179368Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:41:45.90234227Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:41:45.902380331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54101)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902405911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:41:45.902438662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902463752Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902488642Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:52606)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902513853Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.902720855Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.903282482Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.903309213Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.903324573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.903338663Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.903966251Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.904089283Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.904479498Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.904503088Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.904517858Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.904589509Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.904611649Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:45.90462575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.904359196Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55914)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.904602529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.905557891Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.905581991Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906090988Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906651695Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906678876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[0]R flush_numbers[0..12]: [151, 150, 118, 145, 145, 152, 145, 145, 135, 137, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906697416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[0]R generation[0..12]: [74, 74, 56, 71, 71, 75, 71, 71, 64, 66, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906715126Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906733176Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[1]R flush_numbers[0..12]: [151, 150, 118, 145, 145, 152, 145, 145, 135, 137, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906750736Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[1]R generation[0..12]: [74, 74, 56, 71, 71, 75, 71, 71, 64, 66, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906767727Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906791967Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[2]R flush_numbers[0..12]: [151, 150, 118, 145, 145, 152, 145, 145, 135, 137, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906809997Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[2]R generation[0..12]: [74, 74, 56, 71, 71, 75, 71, 71, 64, 66, 72, 74]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-07-10T15:41:55.906827237Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906848908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906865458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906882028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906898368Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906914539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"":"downstairs","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906930969Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906945929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"0","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906962449Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"1","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906978769Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"client":"2","":"downstairs","session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.906995469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.90701085Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"3dfd1ea8-f0d2-414f-b5ee-1eca50b653d2 is now active with session: 88ae22d3-fe9f-4f6b-b621-23599c87f6c9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.9070273Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.90704247Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486,"session_id":"88ae22d3-fe9f-4f6b-b621-23599c87f6c9"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:41:55.90707363Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 8 len 16384 data: 7 6 5 5 02/30 Read block 482 len 36864 03/30 Write block 189 len 4096 data: 4 04/30 Flush 05/30 Flush 06/30 Flush 07/30 Write block 339 len 40960 data: 6 7 8 8 7 7 6 5 6 7 08/30 Read block 517 len 20480 09/30 Read block 219 len 16384 10/30 Read block 44 len 36864 11/30 Write block 549 len 16384 data: 10 12 11 10 12/30 Read block 268 len 16384 13/30 Read block 383 len 12288 14/30 Write block 490 len 12288 data: 5 6 5 15/30 Write block 370 len 36864 data: 12 11 9 10 10 7 9 12 9 16/30 Write block 572 len 28672 data: 4 4 5 5 7 6 7 17/30 Read block 467 len 4096 18/30 Flush 19/30 Write block 136 len 16384 data: 6 8 10 10 20/30 Write block 56 len 12288 data: 7 5 5 21/30 Read block 438 len 12288 22/30 Read block 588 len 20480 23/30 Write block 278 len 20480 data: 7 6 5 5 5 24/30 Read block 415 len 36864 25/30 Read block 496 len 16384 26/30 Read block 249 len 24576 27/30 Write block 472 len 40960 data: 3 6 6 8 10 9 10 8 7 8 28/30 Write block 576 len 28672 data: 8 7 8 7 6 5 5 29/30 Write block 584 len 40960 data: 3 3 5 5 6 6 8 10 8 5 30/30 Read block 398 len 40960 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 10 Done Done Done false 30 Acked 1029 Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 0 1 Done 11 12 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:41:55.933251403Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 74 74 74 150 150 150 F F F 2 040-059 56 56 56 118 118 118 T T T 3 060-079 71 71 71 145 145 145 F F F 4 080-099 71 71 71 145 145 145 F F F 5 100-119 75 75 75 152 152 152 F F F 6 120-139 71 71 71 145 145 145 T T T 7 140-159 71 71 71 145 145 145 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 76 76 76 153 153 153 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 74 74 74 151 151 151 F F F 12 240-259 72 72 72 147 147 147 F F F 13 260-279 73 73 73 149 149 149 T T T 14 280-299 75 75 75 152 152 152 T T T 15 300-319 70 70 70 144 144 144 F F F 16 320-339 76 76 76 156 156 156 F F F 17 340-359 76 76 76 156 156 156 F F F 18 360-379 76 76 76 156 156 156 F F F 19 380-399 74 74 74 151 151 151 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 74 74 74 151 151 151 T T T 24 480-499 76 76 76 156 156 156 T T T 25 500-519 74 74 74 150 150 150 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 76 76 76 156 156 156 F F F 28 560-579 76 76 76 156 156 156 T T T 29 580-599 67 67 67 138 138 138 T T T Max gen: 76, Max flush: 156 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 74 74 74 150 150 150 F F F 2 040-059 77 77 77 157 157 157 F F F 3 060-079 71 71 71 145 145 145 F F F 4 080-099 71 71 71 145 145 145 F F F 5 100-119 75 75 75 152 152 152 F F F 6 120-139 77 77 77 157 157 157 F F F 7 140-159 71 71 71 145 145 145 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 76 76 76 153 153 153 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 74 74 74 151 151 151 F F F 12 240-259 72 72 72 147 147 147 F F F 13 260-279 77 77 77 157 157 157 F F F 14 280-299 77 77 77 157 157 157 F F F 15 300-319 70 70 70 144 144 144 F F F 16 320-339 76 76 76 156 156 156 F F F 17 340-359 76 76 76 156 156 156 F F F 18 360-379 76 76 76 156 156 156 F F F 19 380-399 74 74 74 151 151 151 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 77 77 77 157 157 157 F F F 24 480-499 77 77 77 157 157 157 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 76 76 76 156 156 156 F F F 28 560-579 77 77 77 157 157 157 F F F 29 580-599 77 77 77 157 157 157 F F F Max gen: 77, Max flush: 157 Begin loop 39 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.198500549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.198886134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.198904994Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494} {"msg":"Crucible 2702d49e-d04a-4bb1-bbea-4143a626981e has session id: 50ec1558-f61b-453d-b68b-ee7f07f4a47b","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199294279Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Upstairs opts: Upstairs UUID: 2702d49e-d04a-4bb1-bbea-4143a626981e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199343899Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Crucible stats registered with UUID: 2702d49e-d04a-4bb1-bbea-4143a626981e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.19936575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199650553Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494} {"msg":"2702d49e-d04a-4bb1-bbea-4143a626981e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199714914Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199732055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199746535Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199760545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199774005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199787285Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199800815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199814136Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199828116Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.199841726Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:10.200004028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:10.200047668Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:10.20021527Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.200253221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.200276701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47390)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.200299872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43381)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.200323452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.200345112Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.200816078Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.200854439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.200883439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.20093765Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.201477597Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.201628159Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.201983633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.202005703Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.202024453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.202095965Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.202114845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:10.202128465Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.201650809Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55594)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.201946672Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.202894674Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.202922005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.203467902Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.203978788Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.203996998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[0]R flush_numbers[0..12]: [153, 150, 157, 145, 145, 152, 157, 145, 135, 153, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204011798Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[0]R generation[0..12]: [76, 74, 77, 71, 71, 75, 77, 71, 64, 76, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204025859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204040059Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[1]R flush_numbers[0..12]: [153, 150, 157, 145, 145, 152, 157, 145, 135, 153, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204054069Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[1]R generation[0..12]: [76, 74, 77, 71, 71, 75, 77, 71, 64, 76, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204073589Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204088439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[2]R flush_numbers[0..12]: [153, 150, 157, 145, 145, 152, 157, 145, 135, 153, 147, 151]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.20410248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[2]R generation[0..12]: [76, 74, 77, 71, 71, 75, 77, 71, 64, 76, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.20411652Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.20413077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.2041437Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.20415742Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204174091Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204187231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"":"downstairs","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204200821Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204212991Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"0","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204226461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"1","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204239761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"client":"2","":"downstairs","session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204253412Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204265792Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"2702d49e-d04a-4bb1-bbea-4143a626981e is now active with session: 50ec1558-f61b-453d-b68b-ee7f07f4a47b","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204281812Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204294282Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494,"session_id":"50ec1558-f61b-453d-b68b-ee7f07f4a47b"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:20.204306582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 22 len 8192 data: 5 4 02/30 Read block 276 len 4096 03/30 Write block 479 len 8192 data: 9 8 04/30 Read block 486 len 32768 05/30 Read block 343 len 32768 06/30 Read block 232 len 32768 07/30 Flush 08/30 Write block 146 len 36864 data: 7 9 10 10 7 6 6 5 5 09/30 Read block 281 len 32768 10/30 Read block 462 len 32768 11/30 Read block 271 len 8192 12/30 Write block 234 len 36864 data: 11 11 10 8 6 7 5 6 5 13/30 Read block 289 len 36864 14/30 Write block 302 len 24576 data: 4 5 4 4 4 5 15/30 Read block 594 len 8192 16/30 Flush 17/30 Write block 275 len 8192 data: 4 4 18/30 Read block 169 len 20480 19/30 Write block 285 len 32768 data: 5 5 5 5 5 4 5 6 20/30 Write block 567 len 16384 data: 6 6 3 5 21/30 Write block 386 len 32768 data: 6 4 5 5 6 6 7 8 22/30 Read block 470 len 28672 23/30 Read block 477 len 20480 24/30 Write block 347 len 12288 data: 7 8 5 25/30 Read block 507 len 8192 26/30 Read block 382 len 8192 27/30 Write block 59 len 12288 data: 5 5 6 28/30 Write block 254 len 16384 data: 10 11 8 8 29/30 Read block 368 len 32768 30/30 Read block 236 len 12288 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 2 Done Done Done false 18 Acked 1017 Read 5 Done Done Done false 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Write 4 Done Done Done false 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:20.229842246Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 78 78 78 158 158 158 F F F 2 040-059 77 77 77 157 157 157 T T T 3 060-079 71 71 71 145 145 145 T T T 4 080-099 71 71 71 145 145 145 F F F 5 100-119 75 75 75 152 152 152 F F F 6 120-139 77 77 77 157 157 157 F F F 7 140-159 78 78 78 159 159 159 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 76 76 76 153 153 153 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 78 78 78 159 159 159 F F F 12 240-259 78 78 78 159 159 159 T T T 13 260-279 77 77 77 157 157 157 T T T 14 280-299 77 77 77 157 157 157 T T T 15 300-319 78 78 78 159 159 159 F F F 16 320-339 76 76 76 156 156 156 F F F 17 340-359 76 76 76 156 156 156 T T T 18 360-379 76 76 76 156 156 156 F F F 19 380-399 74 74 74 151 151 151 T T T 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 78 78 78 158 158 158 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 76 76 76 156 156 156 F F F 28 560-579 77 77 77 157 157 157 T T T 29 580-599 77 77 77 157 157 157 F F F Max gen: 78, Max flush: 159 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 78 78 78 158 158 158 F F F 2 040-059 79 79 79 160 160 160 F F F 3 060-079 79 79 79 160 160 160 F F F 4 080-099 71 71 71 145 145 145 F F F 5 100-119 75 75 75 152 152 152 F F F 6 120-139 77 77 77 157 157 157 F F F 7 140-159 78 78 78 159 159 159 F F F 8 160-179 64 64 64 135 135 135 F F F 9 180-199 76 76 76 153 153 153 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 78 78 78 159 159 159 F F F 12 240-259 79 79 79 160 160 160 F F F 13 260-279 79 79 79 160 160 160 F F F 14 280-299 79 79 79 160 160 160 F F F 15 300-319 78 78 78 159 159 159 F F F 16 320-339 76 76 76 156 156 156 F F F 17 340-359 79 79 79 160 160 160 F F F 18 360-379 76 76 76 156 156 156 F F F 19 380-399 79 79 79 160 160 160 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 78 78 78 158 158 158 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 69 69 69 143 143 143 F F F 27 540-559 76 76 76 156 156 156 F F F 28 560-579 79 79 79 160 160 160 F F F 29 580-599 77 77 77 157 157 157 F F F Max gen: 79, Max flush: 160 Begin loop 40 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.49728987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.497663215Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.497702886Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502} {"msg":"Crucible b0a0d649-120c-4206-b008-5d6bd5ec7a4c has session id: 697f9798-941c-4960-80a5-b006ec220bf4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498001569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Upstairs opts: Upstairs UUID: b0a0d649-120c-4206-b008-5d6bd5ec7a4c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.49806208Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Crucible stats registered with UUID: b0a0d649-120c-4206-b008-5d6bd5ec7a4c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498084041Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498409385Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502} {"msg":"b0a0d649-120c-4206-b008-5d6bd5ec7a4c active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498553876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498571717Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498592197Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498617207Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498642028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498664078Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498686748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498711429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498734309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.498749469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:55127)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.499047173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63375)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.499082973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:34.499109483Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:34.499137134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:34.499166244Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.499188615Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.499210535Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.499247225Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.499781482Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.499809682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.499825383Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.499839733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.500450331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.500478191Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.500980777Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.501005318Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.501020348Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.501058498Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.501075728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:34.501089349Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.500614733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63490)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.500743844Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.502115432Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.502148392Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.502692319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503264826Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503293056Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[0]R flush_numbers[0..12]: [153, 158, 160, 160, 145, 152, 157, 159, 135, 153, 147, 159]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503311737Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[0]R generation[0..12]: [76, 78, 79, 79, 71, 75, 77, 78, 64, 76, 72, 78]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503329537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503347577Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[1]R flush_numbers[0..12]: [153, 158, 160, 160, 145, 152, 157, 159, 135, 153, 147, 159]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503370637Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[1]R generation[0..12]: [76, 78, 79, 79, 71, 75, 77, 78, 64, 76, 72, 78]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503388478Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503406338Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[2]R flush_numbers[0..12]: [153, 158, 160, 160, 145, 152, 157, 159, 135, 153, 147, 159]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503423998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R generation[0..12]: [76, 78, 79, 79, 71, 75, 77, 78, 64, 76, 72, 78]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503446118Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503465199Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503481659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503498479Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503514709Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503531059Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"":"downstairs","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.5035481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.5035632Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"0","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.5035806Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"1","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.50359403Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"client":"2","":"downstairs","session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503610051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503622601Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"b0a0d649-120c-4206-b008-5d6bd5ec7a4c is now active with session: 697f9798-941c-4960-80a5-b006ec220bf4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503635601Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503647701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502,"session_id":"697f9798-941c-4960-80a5-b006ec220bf4"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:44.503659831Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 16384 data: 8 6 6 6 02/30 Read block 43 len 4096 03/30 Read block 168 len 20480 04/30 Write block 366 len 16384 data: 4 3 6 8 05/30 Read block 185 len 20480 06/30 Write block 60 len 24576 data: 6 7 4 4 6 9 07/30 Write block 118 len 4096 data: 7 08/30 Read block 234 len 12288 09/30 Read block 187 len 8192 10/30 Read block 507 len 12288 11/30 Read block 54 len 4096 12/30 Flush 13/30 Flush 14/30 Write block 530 len 12288 data: 9 10 9 15/30 Read block 75 len 20480 16/30 Write block 92 len 24576 data: 6 6 6 7 6 6 17/30 Read block 484 len 20480 18/30 Read block 532 len 28672 19/30 Write block 254 len 12288 data: 11 12 9 20/30 Read block 88 len 20480 21/30 Write block 179 len 4096 data: 6 22/30 Read block 477 len 28672 23/30 Flush 24/30 Write block 241 len 28672 data: 7 6 6 8 6 8 9 25/30 Read block 416 len 32768 26/30 Read block 423 len 16384 27/30 Read block 354 len 28672 28/30 Read block 324 len 40960 29/30 Read block 525 len 40960 30/30 Read block 82 len 8192 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 6 6 7 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:44.528923662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 78 78 78 158 158 158 F F F 2 040-059 80 80 80 161 161 161 F F F 3 060-079 80 80 80 161 161 161 F F F 4 080-099 80 80 80 163 163 163 F F F 5 100-119 80 80 80 161 161 161 F F F 6 120-139 77 77 77 157 157 157 F F F 7 140-159 78 78 78 159 159 159 F F F 8 160-179 80 80 80 163 163 163 F F F 9 180-199 76 76 76 153 153 153 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 78 78 78 159 159 159 F F F 12 240-259 80 80 80 163 163 163 T T T 13 260-279 79 79 79 160 160 160 F F F 14 280-299 79 79 79 160 160 160 F F F 15 300-319 78 78 78 159 159 159 F F F 16 320-339 76 76 76 156 156 156 F F F 17 340-359 79 79 79 160 160 160 F F F 18 360-379 80 80 80 161 161 161 F F F 19 380-399 79 79 79 160 160 160 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 78 78 78 158 158 158 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 80 80 80 163 163 163 F F F 27 540-559 76 76 76 156 156 156 F F F 28 560-579 79 79 79 160 160 160 F F F 29 580-599 77 77 77 157 157 157 F F F Max gen: 80, Max flush: 163 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 78 78 78 158 158 158 F F F 2 040-059 80 80 80 161 161 161 F F F 3 060-079 80 80 80 161 161 161 F F F 4 080-099 80 80 80 163 163 163 F F F 5 100-119 80 80 80 161 161 161 F F F 6 120-139 77 77 77 157 157 157 F F F 7 140-159 78 78 78 159 159 159 F F F 8 160-179 80 80 80 163 163 163 F F F 9 180-199 76 76 76 153 153 153 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 78 78 78 159 159 159 F F F 12 240-259 81 81 81 164 164 164 F F F 13 260-279 79 79 79 160 160 160 F F F 14 280-299 79 79 79 160 160 160 F F F 15 300-319 78 78 78 159 159 159 F F F 16 320-339 76 76 76 156 156 156 F F F 17 340-359 79 79 79 160 160 160 F F F 18 360-379 80 80 80 161 161 161 F F F 19 380-399 79 79 79 160 160 160 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 70 70 70 144 144 144 F F F 23 460-479 78 78 78 158 158 158 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 80 80 80 163 163 163 F F F 27 540-559 76 76 76 156 156 156 F F F 28 560-579 79 79 79 160 160 160 F F F 29 580-599 77 77 77 157 157 157 F F F Max gen: 81, Max flush: 164 Begin loop 41 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.70369405Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.704058735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.704096685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510} {"msg":"Crucible a2ff5022-3088-4539-a173-1f07db5a0b56 has session id: 848459df-c943-4803-84c7-1894b0717248","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.704413689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Upstairs opts: Upstairs UUID: a2ff5022-3088-4539-a173-1f07db5a0b56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.70447447Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Crucible stats registered with UUID: a2ff5022-3088-4539-a173-1f07db5a0b56","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.70449855Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.704896395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510} {"msg":"a2ff5022-3088-4539-a173-1f07db5a0b56 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.704923766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.704938726Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.704953416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.704967226Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.704980806Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705009517Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705025637Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705090658Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705109668Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705123618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:54.705313311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:54.705356291Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:42:54.705386892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705408582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705511283Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_connection connected from Ok(127.0.0.1:54947)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705543654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_connection connected from Ok(127.0.0.1:49000)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705571914Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.705596344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.706115181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.706149361Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.706176542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.706203682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.706660418Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.706771039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.707131554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.707165694Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.707190944Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.707309736Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.707330216Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:42:54.707344146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.706543176Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_connection connected from Ok(127.0.0.1:56610)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.706653527Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.707703181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.707729691Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.708347529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.708890636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.708919036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[0]R flush_numbers[0..12]: [153, 158, 161, 161, 163, 161, 157, 159, 163, 153, 147, 159]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.708937826Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[0]R generation[0..12]: [76, 78, 80, 80, 80, 80, 77, 78, 80, 76, 72, 78]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.708955506Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.708973407Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[1]R flush_numbers[0..12]: [153, 158, 161, 161, 163, 161, 157, 159, 163, 153, 147, 159]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.708997317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[1]R generation[0..12]: [76, 78, 80, 80, 80, 80, 77, 78, 80, 76, 72, 78]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709015197Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709033787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[2]R flush_numbers[0..12]: [153, 158, 161, 161, 163, 161, 157, 159, 163, 153, 147, 159]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709051298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[2]R generation[0..12]: [76, 78, 80, 80, 80, 80, 77, 78, 80, 76, 72, 78]","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2024-07-10T15:43:04.709072718Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709090588Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709106928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709123579Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709139849Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709156339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"":"downstairs","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709173099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.7091882Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"0","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.70920507Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"1","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.70922194Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"client":"2","":"downstairs","session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.70924113Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.70925682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"a2ff5022-3088-4539-a173-1f07db5a0b56 is now active with session: 848459df-c943-4803-84c7-1894b0717248","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.70927311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709288441Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510,"session_id":"848459df-c943-4803-84c7-1894b0717248"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:04.709303791Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 223 len 24576 data: 8 10 9 10 11 11 02/30 Read block 194 len 24576 03/30 Read block 83 len 36864 04/30 Read block 569 len 36864 05/30 Read block 557 len 20480 06/30 Read block 153 len 20480 07/30 Flush 08/30 Write block 449 len 20480 data: 5 6 7 8 8 09/30 Write block 448 len 20480 data: 7 6 7 8 9 10/30 Write block 373 len 8192 data: 11 11 11/30 Read block 370 len 16384 12/30 Read block 234 len 28672 13/30 Read block 86 len 8192 14/30 Read block 199 len 16384 15/30 Write block 255 len 8192 data: 13 10 16/30 Read block 51 len 4096 17/30 Write block 272 len 20480 data: 2 2 5 5 5 18/30 Read block 92 len 8192 19/30 Read block 561 len 4096 20/30 Write block 358 len 4096 data: 10 21/30 Read block 200 len 12288 22/30 Read block 593 len 12288 23/30 Write block 177 len 20480 data: 5 5 7 5 4 24/30 Read block 496 len 28672 25/30 Write block 139 len 28672 data: 11 6 8 8 10 9 7 26/30 Write block 332 len 28672 data: 5 4 5 5 6 6 4 27/30 Read block 90 len 36864 28/30 Read block 189 len 40960 29/30 Read block 17 len 36864 30/30 Read block 82 len 12288 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false 31 Acked 1030 Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 8 7 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:43:04.734550331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 78 78 78 158 158 158 F F F 2 040-059 80 80 80 161 161 161 F F F 3 060-079 80 80 80 161 161 161 F F F 4 080-099 80 80 80 163 163 163 F F F 5 100-119 80 80 80 161 161 161 F F F 6 120-139 77 77 77 157 157 157 T T T 7 140-159 78 78 78 159 159 159 T T T 8 160-179 80 80 80 163 163 163 T T T 9 180-199 76 76 76 153 153 153 T T T 10 200-219 72 72 72 147 147 147 F F F 11 220-239 82 82 82 165 165 165 F F F 12 240-259 82 82 82 166 166 166 F F F 13 260-279 82 82 82 166 166 166 F F F 14 280-299 79 79 79 160 160 160 F F F 15 300-319 78 78 78 159 159 159 F F F 16 320-339 76 76 76 156 156 156 T T T 17 340-359 82 82 82 166 166 166 F F F 18 360-379 82 82 82 166 166 166 F F F 19 380-399 79 79 79 160 160 160 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 82 82 82 166 166 166 F F F 23 460-479 78 78 78 158 158 158 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 80 80 80 163 163 163 F F F 27 540-559 76 76 76 156 156 156 F F F 28 560-579 79 79 79 160 160 160 F F F 29 580-599 77 77 77 157 157 157 F F F Max gen: 82, Max flush: 166 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 78 78 78 158 158 158 F F F 2 040-059 80 80 80 161 161 161 F F F 3 060-079 80 80 80 161 161 161 F F F 4 080-099 80 80 80 163 163 163 F F F 5 100-119 80 80 80 161 161 161 F F F 6 120-139 83 83 83 167 167 167 F F F 7 140-159 83 83 83 167 167 167 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 83 83 83 167 167 167 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 82 82 82 165 165 165 F F F 12 240-259 82 82 82 166 166 166 F F F 13 260-279 82 82 82 166 166 166 F F F 14 280-299 79 79 79 160 160 160 F F F 15 300-319 78 78 78 159 159 159 F F F 16 320-339 83 83 83 167 167 167 F F F 17 340-359 82 82 82 166 166 166 F F F 18 360-379 82 82 82 166 166 166 F F F 19 380-399 79 79 79 160 160 160 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 82 82 82 166 166 166 F F F 23 460-479 78 78 78 158 158 158 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 80 80 80 163 163 163 F F F 27 540-559 76 76 76 156 156 156 F F F 28 560-579 79 79 79 160 160 160 F F F 29 580-599 77 77 77 157 157 157 F F F Max gen: 83, Max flush: 167 Begin loop 42 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.959192852Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.959469256Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.959499676Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518} {"msg":"Crucible a6c4e70b-bc85-434e-9a5b-7708f37eca4b has session id: b32fee56-a43a-4987-a3d2-196b5ad16f68","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.959944362Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Upstairs opts: Upstairs UUID: a6c4e70b-bc85-434e-9a5b-7708f37eca4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.959992222Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Crucible stats registered with UUID: a6c4e70b-bc85-434e-9a5b-7708f37eca4b","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960017252Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960369377Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518} {"msg":"a6c4e70b-bc85-434e-9a5b-7708f37eca4b active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960477748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960502798Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960518869Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960533179Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960546909Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960560199Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960573579Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.96062916Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.96064796Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.96066204Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:43:18.960847883Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:43:18.960898734Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:43:18.960928504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.960950054Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.961091976Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_connection connected from Ok(127.0.0.1:60175)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.961128106Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_connection connected from Ok(127.0.0.1:57522)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.961154267Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.961225548Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.961822775Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.961860456Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.961890586Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.961919056Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.962461983Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.962600735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.96294964Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.96299215Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.96301219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.96302801Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.963045351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:18.963062151Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.962595825Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_connection connected from Ok(127.0.0.1:64962)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.962722757Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.964121394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.964147515Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.964680882Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965211448Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965234149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[0]R flush_numbers[0..12]: [153, 158, 161, 161, 163, 161, 167, 167, 167, 167, 147, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965249409Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[0]R generation[0..12]: [76, 78, 80, 80, 80, 80, 83, 83, 83, 83, 72, 82]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965263829Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965278589Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[1]R flush_numbers[0..12]: [153, 158, 161, 161, 163, 161, 167, 167, 167, 167, 147, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965298839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[1]R generation[0..12]: [76, 78, 80, 80, 80, 80, 83, 83, 83, 83, 72, 82]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.96531321Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.9653275Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[2]R flush_numbers[0..12]: [153, 158, 161, 161, 163, 161, 167, 167, 167, 167, 147, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.96534174Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[2]R generation[0..12]: [76, 78, 80, 80, 80, 80, 83, 83, 83, 83, 72, 82]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.96535551Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.96536954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.96538242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"}Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.96539984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965412811Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965425861Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"":"downstairs","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965439301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965451431Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"0","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965464671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"1","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965477821Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"client":"2","":"downstairs","session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965493422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965506042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"a6c4e70b-bc85-434e-9a5b-7708f37eca4b is now active with session: b32fee56-a43a-4987-a3d2-196b5ad16f68","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965518982Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965530962Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518,"session_id":"b32fee56-a43a-4987-a3d2-196b5ad16f68"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:28.965543232Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 462 len 20480 data: 4 5 5 4 3 02/30 Read block 246 len 40960 03/30 Read block 250 len 12288 04/30 Read block 420 len 16384 05/30 Read block 412 len 20480 06/30 Read block 475 len 20480 07/30 Write block 375 len 24576 data: 8 10 13 10 8 7 08/30 Read block 569 len 40960 09/30 Write block 583 len 16384 data: 5 4 4 6 10/30 Write block 557 len 40960 data: 8 6 3 2 3 5 4 5 5 5 11/30 Read block 506 len 4096 12/30 Read block 167 len 12288 13/30 Read block 43 len 12288 14/30 Flush 15/30 Write block 385 len 28672 data: 5 7 5 6 6 7 7 16/30 Read block 365 len 24576 17/30 Read block 192 len 28672 18/30 Read block 218 len 16384 19/30 Flush 20/30 Write block 528 len 8192 data: 9 8 21/30 Write block 523 len 20480 data: 3 4 5 10 9 22/30 Write block 578 len 24576 data: 9 8 7 6 6 6 23/30 Write block 42 len 32768 data: 9 9 8 7 7 5 4 4 24/30 Flush 25/30 Write block 465 len 16384 data: 5 4 3 4 26/30 Write block 122 len 40960 data: 6 4 4 4 3 4 4 6 10 8 27/30 Read block 352 len 16384 28/30 Read block 570 len 12288 29/30 Write block 385 len 8192 data: 6 8 30/30 Read block 530 len 20480 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 2 Done Done Done false 31 Acked 1030 Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 1 1 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:43:28.995055367Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 78 78 78 158 158 158 F F F 2 040-059 84 84 84 171 171 171 F F F 3 060-079 80 80 80 161 161 161 F F F 4 080-099 80 80 80 163 163 163 F F F 5 100-119 80 80 80 161 161 161 F F F 6 120-139 83 83 83 167 167 167 T T T 7 140-159 83 83 83 167 167 167 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 83 83 83 167 167 167 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 82 82 82 165 165 165 F F F 12 240-259 82 82 82 166 166 166 F F F 13 260-279 82 82 82 166 166 166 F F F 14 280-299 79 79 79 160 160 160 F F F 15 300-319 78 78 78 159 159 159 F F F 16 320-339 83 83 83 167 167 167 F F F 17 340-359 82 82 82 166 166 166 F F F 18 360-379 84 84 84 168 168 168 F F F 19 380-399 84 84 84 169 169 169 T T T 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 82 82 82 166 166 166 F F F 23 460-479 84 84 84 168 168 168 T T T 24 480-499 78 78 78 158 158 158 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 84 84 84 171 171 171 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 84 84 84 171 171 171 F F F 29 580-599 84 84 84 171 171 171 F F F Max gen: 84, Max flush: 171 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 F F F 1 020-039 78 78 78 158 158 158 F F F 2 040-059 84 84 84 171 171 171 F F F 3 060-079 80 80 80 161 161 161 F F F 4 080-099 80 80 80 163 163 163 F F F 5 100-119 80 80 80 161 161 161 F F F 6 120-139 85 85 85 172 172 172 F F F 7 140-159 83 83 83 167 167 167 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 83 83 83 167 167 167 F F F 10 200-219 72 72 72 147 147 147 F F F 11 220-239 82 82 82 165 165 165 F F F 12 240-259 82 82 82 166 166 166 F F F 13 260-279 82 82 82 166 166 166 F F F 14 280-299 79 79 79 160 160 160 F F F 15 300-319 78 78 78 159 159 159 F F F 16 320-339 83 83 83 167 167 167 F F F 17 340-359 82 82 82 166 166 166 F F F 18 360-379 84 84 84 168 168 168 F F F 19 380-399 85 85 85 172 172 172 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 82 82 82 166 166 166 F F F 23 460-479 85 85 85 172 172 172 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 74 74 74 150 150 150 F F F 26 520-539 84 84 84 171 171 171 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 84 84 84 171 171 171 F F F 29 580-599 84 84 84 171 171 171 F F F Max gen: 85, Max flush: 172 Begin loop 43 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.198130424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.198506119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.198541379Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526} {"msg":"Crucible e629f45f-2f01-4b09-a41e-e2985870a3ae has session id: fe9e9164-e439-4643-acb7-d8e425c9bc04","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.198960095Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Upstairs opts: Upstairs UUID: e629f45f-2f01-4b09-a41e-e2985870a3ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199005395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Crucible stats registered with UUID: e629f45f-2f01-4b09-a41e-e2985870a3ae","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199028836Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199296929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526} {"msg":"e629f45f-2f01-4b09-a41e-e2985870a3ae active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199420601Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199445911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199462101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199476271Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199490411Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199504262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199517942Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199531402Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199545292Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199559052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:43:43.199777045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_connection connected from Ok(127.0.0.1:39715)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199893456Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_connection connected from Ok(127.0.0.1:64931)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199911587Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:43:43.199928277Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:43:43.199944807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199957427Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.199970648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.200087629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.200544135Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.200578325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.200600816Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.200622746Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.201239104Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.201275964Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.20176486Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.201789921Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.201805411Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.201831851Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.201845911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:43.201859872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.201111512Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_connection connected from Ok(127.0.0.1:43407)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.201264434Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.202295647Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.202329698Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.202994986Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203608794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203636454Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[0]R flush_numbers[0..12]: [153, 158, 171, 161, 163, 161, 172, 167, 167, 167, 147, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203655424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[0]R generation[0..12]: [76, 78, 84, 80, 80, 80, 85, 83, 83, 83, 72, 82]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203673584Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203691755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[1]R flush_numbers[0..12]: [153, 158, 171, 161, 163, 161, 172, 167, 167, 167, 147, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203714495Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[1]R generation[0..12]: [76, 78, 84, 80, 80, 80, 85, 83, 83, 83, 72, 82]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203732125Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203750005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[2]R flush_numbers[0..12]: [153, 158, 171, 161, 163, 161, 172, 167, 167, 167, 147, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203768386Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[2]R generation[0..12]: [76, 78, 84, 80, 80, 80, 85, 83, 83, 83, 72, 82]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:43:53.203786346Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203804606Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203827717Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203852697Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203875947Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203895737Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"":"downstairs","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203918727Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203945278Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"0","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.203975778Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"1","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.204005099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"client":"2","":"downstairs","session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.204029879Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.204051049Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"e629f45f-2f01-4b09-a41e-e2985870a3ae is now active with session: fe9e9164-e439-4643-acb7-d8e425c9bc04","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.20406788Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.20408308Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526,"session_id":"fe9e9164-e439-4643-acb7-d8e425c9bc04"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:43:53.20409867Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 455 len 8192 data: 5 4 03/30 Write block 461 len 32768 data: 4 5 6 6 6 5 4 5 04/30 Read block 9 len 24576 05/30 Read block 341 len 20480 06/30 Read block 112 len 28672 07/30 Flush 08/30 Write block 197 len 20480 data: 8 6 5 5 4 09/30 Read block 528 len 4096 10/30 Write block 590 len 28672 data: 9 11 9 6 6 5 4 11/30 Read block 219 len 8192 12/30 Read block 309 len 4096 13/30 Write block 516 len 28672 data: 3 3 4 3 3 3 3 14/30 Read block 422 len 24576 15/30 Read block 323 len 12288 16/30 Read block 18 len 4096 17/30 Write block 43 len 40960 data: 10 9 8 8 6 5 5 5 7 9 18/30 Read block 237 len 8192 19/30 Read block 190 len 40960 20/30 Write block 49 len 40960 data: 6 6 8 10 8 6 6 9 7 7 21/30 Read block 76 len 20480 22/30 Read block 452 len 28672 23/30 Read block 189 len 20480 24/30 Read block 480 len 8192 25/30 Read block 322 len 32768 26/30 Read block 451 len 40960 27/30 Read block 153 len 20480 28/30 Read block 423 len 40960 29/30 Read block 429 len 24576 30/30 Write block 16 len 28672 data: 6 6 8 8 7 7 6 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 10 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:43:53.23010357Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 153 153 153 T T T 1 020-039 78 78 78 158 158 158 T T T 2 040-059 86 86 86 175 175 175 T T T 3 060-079 80 80 80 161 161 161 F F F 4 080-099 80 80 80 163 163 163 F F F 5 100-119 80 80 80 161 161 161 F F F 6 120-139 85 85 85 172 172 172 F F F 7 140-159 83 83 83 167 167 167 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 86 86 86 175 175 175 F F F 10 200-219 86 86 86 175 175 175 F F F 11 220-239 82 82 82 165 165 165 F F F 12 240-259 82 82 82 166 166 166 F F F 13 260-279 82 82 82 166 166 166 F F F 14 280-299 79 79 79 160 160 160 F F F 15 300-319 78 78 78 159 159 159 F F F 16 320-339 83 83 83 167 167 167 F F F 17 340-359 82 82 82 166 166 166 F F F 18 360-379 84 84 84 168 168 168 F F F 19 380-399 85 85 85 172 172 172 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 86 86 86 174 174 174 F F F 23 460-479 86 86 86 174 174 174 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 86 86 86 175 175 175 F F F 26 520-539 86 86 86 175 175 175 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 84 84 84 171 171 171 F F F 29 580-599 86 86 86 175 175 175 F F F Max gen: 86, Max flush: 175 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 87 87 87 176 176 176 F F F 2 040-059 87 87 87 176 176 176 F F F 3 060-079 80 80 80 161 161 161 F F F 4 080-099 80 80 80 163 163 163 F F F 5 100-119 80 80 80 161 161 161 F F F 6 120-139 85 85 85 172 172 172 F F F 7 140-159 83 83 83 167 167 167 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 86 86 86 175 175 175 F F F 10 200-219 86 86 86 175 175 175 F F F 11 220-239 82 82 82 165 165 165 F F F 12 240-259 82 82 82 166 166 166 F F F 13 260-279 82 82 82 166 166 166 F F F 14 280-299 79 79 79 160 160 160 F F F 15 300-319 78 78 78 159 159 159 F F F 16 320-339 83 83 83 167 167 167 F F F 17 340-359 82 82 82 166 166 166 F F F 18 360-379 84 84 84 168 168 168 F F F 19 380-399 85 85 85 172 172 172 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 70 70 70 144 144 144 F F F 22 440-459 86 86 86 174 174 174 F F F 23 460-479 86 86 86 174 174 174 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 86 86 86 175 175 175 F F F 26 520-539 86 86 86 175 175 175 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 84 84 84 171 171 171 F F F 29 580-599 86 86 86 175 175 175 F F F Max gen: 87, Max flush: 176 Begin loop 44 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.428858072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.429143736Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.429166006Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534} {"msg":"Crucible aa79b1cf-021c-4aa8-9ebc-1e15e3b062c7 has session id: 26448f13-c9a7-4158-8737-289f63369536","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.429642562Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Upstairs opts: Upstairs UUID: aa79b1cf-021c-4aa8-9ebc-1e15e3b062c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.429677683Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Crucible stats registered with UUID: aa79b1cf-021c-4aa8-9ebc-1e15e3b062c7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.429693873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430035567Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534} {"msg":"aa79b1cf-021c-4aa8-9ebc-1e15e3b062c7 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430064368Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430088508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430113368Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430135689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430159129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430184349Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430206649Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.43022934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.43025282Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.4302819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_connection connected from Ok(127.0.0.1:32984)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430331461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_connection connected from Ok(127.0.0.1:61278)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430610564Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:07.430650205Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:07.430691245Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:07.430717676Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430738046Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430758946Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.430872458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.431130311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.431158331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.431292673Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.431344104Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.431688778Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.43182242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.432193604Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.432221045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.432236165Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.432249685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.432263565Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:07.432276766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.432530689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_connection connected from Ok(127.0.0.1:61744)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.432751151Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.433751964Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.433784315Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.434327251Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.434850668Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.434877629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[0]R flush_numbers[0..12]: [176, 176, 176, 161, 163, 161, 172, 167, 167, 175, 175, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.434896639Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[0]R generation[0..12]: [87, 87, 87, 80, 80, 80, 85, 83, 83, 86, 86, 82]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.434914529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.434937409Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[1]R flush_numbers[0..12]: [176, 176, 176, 161, 163, 161, 172, 167, 167, 175, 175, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.43495585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[1]R generation[0..12]: [87, 87, 87, 80, 80, 80, 85, 83, 83, 86, 86, 82]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.43497355Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.43499135Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[2]R flush_numbers[0..12]: [176, 176, 176, 161, 163, 161, 172, 167, 167, 175, 175, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.43500889Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[2]R generation[0..12]: [87, 87, 87, 80, 80, 80, 85, 83, 83, 86, 86, 82]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.43503018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435047911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435064301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435080921Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435097251Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435113361Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"":"downstairs","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435129932Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435144952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"0","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435161492Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"1","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435180462Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"client":"2","":"downstairs","session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435197433Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435212873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"aa79b1cf-021c-4aa8-9ebc-1e15e3b062c7 is now active with session: 26448f13-c9a7-4158-8737-289f63369536","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435229013Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435244153Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"26448f13-c9a7-4158-8737-289f63369536"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:17.435264253Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 8192 data: 7 7 02/30 Write block 294 len 4096 data: 7 03/30 Read block 39 len 32768 04/30 Read block 173 len 8192 05/30 Write block 235 len 20480 data: 12 11 9 7 8 06/30 Write block 580 len 8192 data: 8 7 07/30 Read block 59 len 32768 08/30 Write block 441 len 28672 data: 12 10 9 8 6 7 8 09/30 Read block 298 len 20480 10/30 Write block 523 len 28672 data: 4 5 6 11 10 10 9 11/30 Write block 113 len 4096 data: 7 12/30 Read block 196 len 32768 13/30 Flush 14/30 Write block 254 len 12288 data: 12 14 11 15/30 Write block 422 len 12288 data: 6 7 7 16/30 Write block 529 len 40960 data: 10 10 11 10 6 6 5 7 9 10 17/30 Write block 77 len 36864 data: 5 5 5 4 4 4 5 5 6 18/30 Read block 209 len 24576 19/30 Write block 334 len 4096 data: 6 20/30 Read block 398 len 12288 21/30 Read block 253 len 36864 22/30 Read block 36 len 8192 23/30 Read block 588 len 28672 24/30 Read block 102 len 32768 25/30 Read block 443 len 24576 26/30 Read block 38 len 32768 27/30 Write block 281 len 16384 data: 6 6 5 4 28/30 Read block 200 len 36864 29/30 Write block 240 len 40960 data: 6 8 7 7 9 7 9 10 12 11 30/30 Read block 228 len 20480 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 10 Done Sent Sent false 31 Acked 1030 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 2 2 4 Done 9 7 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:17.460390182Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 87 87 87 176 176 176 F F F 2 040-059 87 87 87 176 176 176 F F F 3 060-079 88 88 88 178 178 178 F F F 4 080-099 88 88 88 178 178 178 F F F 5 100-119 88 88 88 177 177 177 F F F 6 120-139 85 85 85 172 172 172 F F F 7 140-159 83 83 83 167 167 167 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 86 86 86 175 175 175 F F F 10 200-219 86 86 86 175 175 175 F F F 11 220-239 88 88 88 177 177 177 F F F 12 240-259 88 88 88 178 178 178 T T T 13 260-279 82 82 82 166 166 166 F F F 14 280-299 88 88 88 177 177 177 T T T 15 300-319 88 88 88 177 177 177 F F F 16 320-339 88 88 88 178 178 178 F F F 17 340-359 82 82 82 166 166 166 F F F 18 360-379 84 84 84 168 168 168 F F F 19 380-399 85 85 85 172 172 172 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 88 88 88 178 178 178 F F F 22 440-459 88 88 88 177 177 177 F F F 23 460-479 86 86 86 174 174 174 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 86 86 86 175 175 175 F F F 26 520-539 88 88 88 178 178 178 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 84 84 84 171 171 171 F F F 29 580-599 88 88 88 177 177 177 F F F Max gen: 88, Max flush: 178 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 87 87 87 176 176 176 F F F 2 040-059 87 87 87 176 176 176 F F F 3 060-079 88 88 88 178 178 178 F F F 4 080-099 88 88 88 178 178 178 F F F 5 100-119 88 88 88 177 177 177 F F F 6 120-139 85 85 85 172 172 172 F F F 7 140-159 83 83 83 167 167 167 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 86 86 86 175 175 175 F F F 10 200-219 86 86 86 175 175 175 F F F 11 220-239 88 88 88 177 177 177 F F F 12 240-259 89 89 89 179 179 179 F F F 13 260-279 82 82 82 166 166 166 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 88 88 88 177 177 177 F F F 16 320-339 88 88 88 178 178 178 F F F 17 340-359 82 82 82 166 166 166 F F F 18 360-379 84 84 84 168 168 168 F F F 19 380-399 85 85 85 172 172 172 F F F 20 400-419 74 74 74 151 151 151 F F F 21 420-439 88 88 88 178 178 178 F F F 22 440-459 88 88 88 177 177 177 F F F 23 460-479 86 86 86 174 174 174 F F F 24 480-499 78 78 78 158 158 158 F F F 25 500-519 86 86 86 175 175 175 F F F 26 520-539 88 88 88 178 178 178 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 84 84 84 171 171 171 F F F 29 580-599 88 88 88 177 177 177 F F F Max gen: 89, Max flush: 179 Begin loop 45 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.651219684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.651631159Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.651655039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542} {"msg":"Crucible 3bb9f0a7-f3fb-4a87-8ddd-7a809f418f6d has session id: 11962055-fe1a-4267-8627-fc14ff3efb8a","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652003904Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Upstairs opts: Upstairs UUID: 3bb9f0a7-f3fb-4a87-8ddd-7a809f418f6d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652048904Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Crucible stats registered with UUID: 3bb9f0a7-f3fb-4a87-8ddd-7a809f418f6d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652065045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652365718Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542} {"msg":"3bb9f0a7-f3fb-4a87-8ddd-7a809f418f6d active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652387649Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652401079Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652415539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652429469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652443169Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652456459Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.65247007Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652545931Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652581331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652600431Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57441)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652733633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:31.652908025Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:31.652940516Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:31.652958596Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652971506Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.652984926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39919)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.653003526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.653163179Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.653506633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.653532943Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.653547483Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.653561173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.654161001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.654192432Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.654754239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.654783009Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.654798819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.65481282Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.65482678Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:31.65484001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.654141521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58009)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.654420195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.655394917Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.655428997Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.655905093Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656466221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656491171Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[0]R flush_numbers[0..12]: [176, 176, 176, 178, 178, 177, 172, 167, 167, 175, 175, 177]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656510261Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[0]R generation[0..12]: [87, 87, 87, 88, 88, 88, 85, 83, 83, 86, 86, 88]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656533961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656552791Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[1]R flush_numbers[0..12]: [176, 176, 176, 178, 178, 177, 172, 167, 167, 175, 175, 177]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656570582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[1]R generation[0..12]: [87, 87, 87, 88, 88, 88, 85, 83, 83, 86, 86, 88]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656588012Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656606012Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[2]R flush_numbers[0..12]: [176, 176, 176, 178, 178, 177, 172, 167, 167, 175, 175, 177]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656623592Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[2]R generation[0..12]: [87, 87, 87, 88, 88, 88, 85, 83, 83, 86, 86, 88]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656640713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:44:41.656658573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656676933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656693793Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656710084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656726534Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"":"downstairs","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656743604Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656758544Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"0","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656779254Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"1","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656796575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"client":"2","":"downstairs","session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656813775Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656829325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"3bb9f0a7-f3fb-4a87-8ddd-7a809f418f6d is now active with session: 11962055-fe1a-4267-8627-fc14ff3efb8a","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656845725Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656860935Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542,"session_id":"11962055-fe1a-4267-8627-fc14ff3efb8a"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:41.656876226Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 253 len 20480 data: 10 13 15 12 9 02/30 Flush 03/30 Write block 534 len 4096 data: 7 04/30 Write block 480 len 8192 data: 9 9 05/30 Read block 203 len 16384 06/30 Write block 196 len 12288 data: 7 9 7 07/30 Write block 344 len 16384 data: 8 7 6 8 08/30 Write block 563 len 8192 data: 5 6 09/30 Read block 26 len 4096 10/30 Read block 179 len 8192 11/30 Read block 331 len 24576 12/30 Read block 274 len 36864 13/30 Read block 324 len 28672 14/30 Read block 332 len 24576 15/30 Read block 260 len 8192 16/30 Read block 535 len 40960 17/30 Read block 23 len 40960 18/30 Read block 419 len 36864 19/30 Read block 570 len 40960 20/30 Write block 374 len 40960 data: 12 9 11 14 11 9 8 7 7 4 21/30 Write block 386 len 8192 data: 9 6 22/30 Write block 209 len 40960 data: 7 8 8 7 6 5 5 5 4 3 23/30 Write block 359 len 12288 data: 9 8 8 24/30 Read block 127 len 4096 25/30 Write block 476 len 16384 data: 11 10 11 10 26/30 Read block 398 len 16384 27/30 Write block 395 len 24576 data: 7 8 8 8 9 9 28/30 Read block 404 len 36864 29/30 Read block 104 len 24576 30/30 Write block 71 len 40960 data: 6 5 6 6 6 5 6 6 6 5 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:41.680628627Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 87 87 87 176 176 176 F F F 2 040-059 87 87 87 176 176 176 F F F 3 060-079 88 88 88 178 178 178 T T T 4 080-099 88 88 88 178 178 178 T T T 5 100-119 88 88 88 177 177 177 F F F 6 120-139 85 85 85 172 172 172 F F F 7 140-159 83 83 83 167 167 167 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 90 90 90 181 181 181 F F F 10 200-219 86 86 86 175 175 175 T T T 11 220-239 88 88 88 177 177 177 F F F 12 240-259 90 90 90 180 180 180 F F F 13 260-279 82 82 82 166 166 166 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 88 88 88 177 177 177 F F F 16 320-339 88 88 88 178 178 178 F F F 17 340-359 90 90 90 181 181 181 T T T 18 360-379 90 90 90 181 181 181 T T T 19 380-399 90 90 90 181 181 181 T T T 20 400-419 74 74 74 151 151 151 T T T 21 420-439 88 88 88 178 178 178 F F F 22 440-459 88 88 88 177 177 177 F F F 23 460-479 86 86 86 174 174 174 T T T 24 480-499 90 90 90 181 181 181 F F F 25 500-519 86 86 86 175 175 175 F F F 26 520-539 90 90 90 181 181 181 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 90 90 90 181 181 181 F F F 29 580-599 88 88 88 177 177 177 F F F Max gen: 90, Max flush: 181 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 87 87 87 176 176 176 F F F 2 040-059 87 87 87 176 176 176 F F F 3 060-079 91 91 91 182 182 182 F F F 4 080-099 91 91 91 182 182 182 F F F 5 100-119 88 88 88 177 177 177 F F F 6 120-139 85 85 85 172 172 172 F F F 7 140-159 83 83 83 167 167 167 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 90 90 90 181 181 181 F F F 10 200-219 91 91 91 182 182 182 F F F 11 220-239 88 88 88 177 177 177 F F F 12 240-259 90 90 90 180 180 180 F F F 13 260-279 82 82 82 166 166 166 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 88 88 88 177 177 177 F F F 16 320-339 88 88 88 178 178 178 F F F 17 340-359 91 91 91 182 182 182 F F F 18 360-379 91 91 91 182 182 182 F F F 19 380-399 91 91 91 182 182 182 F F F 20 400-419 91 91 91 182 182 182 F F F 21 420-439 88 88 88 178 178 178 F F F 22 440-459 88 88 88 177 177 177 F F F 23 460-479 91 91 91 182 182 182 F F F 24 480-499 90 90 90 181 181 181 F F F 25 500-519 86 86 86 175 175 175 F F F 26 520-539 90 90 90 181 181 181 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 90 90 90 181 181 181 F F F 29 580-599 88 88 88 177 177 177 F F F Max gen: 91, Max flush: 182 Begin loop 46 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.942980056Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.94331227Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.94333525Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550} {"msg":"Crucible d8861fa3-de8c-4864-a2b1-9235fd8301f1 has session id: 0e7d23b4-2a30-4a2e-b244-38e78ea4721a","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.943784886Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Upstairs opts: Upstairs UUID: d8861fa3-de8c-4864-a2b1-9235fd8301f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.943828947Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Crucible stats registered with UUID: d8861fa3-de8c-4864-a2b1-9235fd8301f1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.943847017Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944231802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550} {"msg":"d8861fa3-de8c-4864-a2b1-9235fd8301f1 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944304543Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944322493Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944337233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944358794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944383984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944409314Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944431984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944455485Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944479985Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944504485Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33853)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944622117Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33265)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.944660767Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:55.944967171Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:55.945027932Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:44:55.945062812Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.945086633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.945112003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.945167094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.945398787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.945421407Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.945521818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.945540898Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.945882733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.946006734Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.9464697Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.946504871Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.946527891Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.946548901Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.946570562Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:44:55.946591932Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.946280538Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59744)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.946373319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.947484933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.947532084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.94803664Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948570637Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948599437Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[0]R flush_numbers[0..12]: [176, 176, 176, 182, 182, 177, 172, 167, 167, 181, 182, 177]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948624818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[0]R generation[0..12]: [87, 87, 87, 91, 91, 88, 85, 83, 83, 90, 91, 88]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948643638Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948663188Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[1]R flush_numbers[0..12]: [176, 176, 176, 182, 182, 177, 172, 167, 167, 181, 182, 177]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948681048Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[1]R generation[0..12]: [87, 87, 87, 91, 91, 88, 85, 83, 83, 90, 91, 88]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948701098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948730869Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R flush_numbers[0..12]: [176, 176, 176, 182, 182, 177, 172, 167, 167, 181, 182, 177]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948762489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[2]R generation[0..12]: [87, 87, 87, 91, 91, 88, 85, 83, 83, 90, 91, 88]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.94879054Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.94881523Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.94883307Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.94885334Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948876871Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948901241Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"":"downstairs","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948923621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948940162Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"0","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948958832Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"1","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.948981662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"client":"2","":"downstairs","session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.949006813Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.949026413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"d8861fa3-de8c-4864-a2b1-9235fd8301f1 is now active with session: 0e7d23b4-2a30-4a2e-b244-38e78ea4721a","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.949040213Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.949052323Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550,"session_id":"0e7d23b4-2a30-4a2e-b244-38e78ea4721a"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:05.949071903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 218 len 20480 data: 4 5 5 8 9 04/30 Read block 581 len 12288 05/30 Read block 573 len 36864 06/30 Flush 07/30 Write block 123 len 40960 data: 5 5 5 4 5 5 7 11 9 7 08/30 Read block 511 len 4096 09/30 Read block 52 len 16384 10/30 Write block 585 len 24576 data: 5 7 6 7 7 10 11/30 Write block 236 len 40960 data: 12 10 8 9 7 9 8 8 10 8 12/30 Read block 516 len 24576 13/30 Flush 14/30 Write block 259 len 36864 data: 8 9 7 6 5 4 4 4 2 15/30 Write block 490 len 32768 data: 6 7 6 6 7 6 7 7 16/30 Read block 397 len 36864 17/30 Read block 42 len 12288 18/30 Read block 273 len 16384 19/30 Write block 219 len 36864 data: 6 6 9 10 9 11 10 11 12 20/30 Read block 129 len 36864 21/30 Write block 151 len 32768 data: 7 7 6 6 5 6 5 8 22/30 Flush 23/30 Flush 24/30 Write block 260 len 32768 data: 10 8 7 6 5 5 5 3 25/30 Flush 26/30 Write block 114 len 28672 data: 7 8 8 7 8 7 8 27/30 Write block 437 len 36864 data: 7 7 8 8 13 11 10 9 7 28/30 Write block 348 len 24576 data: 9 6 6 7 8 9 29/30 Read block 326 len 24576 30/30 Write block 447 len 40960 data: 9 8 7 8 9 10 9 6 6 5 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Write 6 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:45:05.980573643Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 87 87 87 176 176 176 F F F 2 040-059 87 87 87 176 176 176 F F F 3 060-079 91 91 91 182 182 182 F F F 4 080-099 91 91 91 182 182 182 F F F 5 100-119 88 88 88 177 177 177 T T T 6 120-139 92 92 92 186 186 186 T T T 7 140-159 92 92 92 188 188 188 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 90 90 90 181 181 181 F F F 10 200-219 92 92 92 188 188 188 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 92 92 92 187 187 187 F F F 13 260-279 92 92 92 190 190 190 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 88 88 88 177 177 177 F F F 16 320-339 88 88 88 178 178 178 F F F 17 340-359 91 91 91 182 182 182 T T T 18 360-379 91 91 91 182 182 182 F F F 19 380-399 91 91 91 182 182 182 F F F 20 400-419 91 91 91 182 182 182 F F F 21 420-439 88 88 88 178 178 178 T T T 22 440-459 88 88 88 177 177 177 T T T 23 460-479 91 91 91 182 182 182 F F F 24 480-499 92 92 92 188 188 188 F F F 25 500-519 86 86 86 175 175 175 F F F 26 520-539 90 90 90 181 181 181 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 90 90 90 181 181 181 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 92, Max flush: 190 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 87 87 87 176 176 176 F F F 2 040-059 87 87 87 176 176 176 F F F 3 060-079 91 91 91 182 182 182 F F F 4 080-099 91 91 91 182 182 182 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 93 93 93 191 191 191 F F F 7 140-159 92 92 92 188 188 188 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 90 90 90 181 181 181 F F F 10 200-219 92 92 92 188 188 188 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 92 92 92 187 187 187 F F F 13 260-279 92 92 92 190 190 190 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 88 88 88 177 177 177 F F F 16 320-339 88 88 88 178 178 178 F F F 17 340-359 93 93 93 191 191 191 F F F 18 360-379 91 91 91 182 182 182 F F F 19 380-399 91 91 91 182 182 182 F F F 20 400-419 91 91 91 182 182 182 F F F 21 420-439 93 93 93 191 191 191 F F F 22 440-459 93 93 93 191 191 191 F F F 23 460-479 91 91 91 182 182 182 F F F 24 480-499 92 92 92 188 188 188 F F F 25 500-519 86 86 86 175 175 175 F F F 26 520-539 90 90 90 181 181 181 F F F 27 540-559 84 84 84 168 168 168 F F F 28 560-579 90 90 90 181 181 181 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 93, Max flush: 191 Begin loop 47 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.209399587Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.209679681Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.209697481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558} {"msg":"Crucible 59bd003b-ff7f-4ab5-b718-96228c00cbf7 has session id: 92df111d-09e7-4f49-b31b-c70d350b96ee","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210116316Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Upstairs opts: Upstairs UUID: 59bd003b-ff7f-4ab5-b718-96228c00cbf7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210163537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Crucible stats registered with UUID: 59bd003b-ff7f-4ab5-b718-96228c00cbf7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210181467Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.21043715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558} {"msg":"59bd003b-ff7f-4ab5-b718-96228c00cbf7 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210467671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210490141Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210514311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210553152Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210568802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210583172Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210602942Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210620862Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210641393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210655913Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:57263)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.210877776Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:46999)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.211009217Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:45:20.211117879Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:45:20.211161399Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:45:20.211236Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.211260581Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.211285231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.211325721Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.211743907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.211781617Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.211810428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.211868748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.212442785Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.212470496Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.212972682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.212998603Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.213013313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.213026023Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.213039703Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:20.213052773Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.213369988Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:54826)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.213516739Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.214428351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.214457621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215061889Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215648097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215667847Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[0]R flush_numbers[0..12]: [176, 176, 176, 182, 182, 191, 191, 188, 167, 181, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215688207Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[0]R generation[0..12]: [87, 87, 87, 91, 91, 93, 93, 92, 83, 90, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215703857Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215718487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[1]R flush_numbers[0..12]: [176, 176, 176, 182, 182, 191, 191, 188, 167, 181, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215733138Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[1]R generation[0..12]: [87, 87, 87, 91, 91, 93, 93, 92, 83, 90, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215746978Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215761208Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[2]R flush_numbers[0..12]: [176, 176, 176, 182, 182, 191, 191, 188, 167, 181, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215775338Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[2]R generation[0..12]: [87, 87, 87, 91, 91, 93, 93, 92, 83, 90, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215789098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215803228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215816319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215832849Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215846099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215859159Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"":"downstairs","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215872409Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215893989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"0","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.21591404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"1","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.21593322Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"client":"2","":"downstairs","session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.21595261Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215965681Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"59bd003b-ff7f-4ab5-b718-96228c00cbf7 is now active with session: 92df111d-09e7-4f49-b31b-c70d350b96ee","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.215986761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.216008621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558,"session_id":"92df111d-09e7-4f49-b31b-c70d350b96ee"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:30.216030671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 52 len 40960 data: 11 9 7 7 10 8 8 7 7 8 02/30 Write block 338 len 40960 data: 5 7 8 9 9 8 9 8 7 9 03/30 Read block 173 len 4096 04/30 Read block 139 len 8192 05/30 Read block 262 len 40960 06/30 Flush 07/30 Write block 26 len 16384 data: 4 4 5 6 08/30 Write block 546 len 12288 data: 6 5 7 09/30 Read block 406 len 36864 10/30 Read block 171 len 40960 11/30 Write block 194 len 32768 data: 7 7 8 10 8 6 6 5 12/30 Read block 81 len 12288 13/30 Write block 131 len 28672 data: 10 8 6 8 8 7 9 14/30 Read block 576 len 8192 15/30 Write block 180 len 4096 data: 6 16/30 Write block 424 len 24576 data: 8 7 6 7 7 7 17/30 Read block 315 len 24576 18/30 Flush 19/30 Write block 533 len 36864 data: 7 8 6 8 10 11 8 7 5 20/30 Write block 83 len 12288 data: 6 6 7 21/30 Write block 476 len 12288 data: 12 11 12 22/30 Read block 247 len 8192 23/30 Write block 243 len 36864 data: 9 11 9 10 11 13 12 12 13 24/30 Write block 566 len 12288 data: 6 7 7 25/30 Read block 495 len 16384 26/30 Write block 314 len 16384 data: 4 4 5 6 27/30 Read block 31 len 12288 28/30 Read block 231 len 36864 29/30 Write block 497 len 28672 data: 8 8 9 10 10 10 11 30/30 Read block 311 len 4096 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 9 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 7 Done Done Sent false 30 Acked 1029 Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 2 2 Done 12 12 10 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:45:30.242438646Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 94 94 94 193 193 193 F F F 2 040-059 94 94 94 192 192 192 F F F 3 060-079 94 94 94 192 192 192 F F F 4 080-099 91 91 91 182 182 182 T T T 5 100-119 93 93 93 191 191 191 F F F 6 120-139 94 94 94 193 193 193 F F F 7 140-159 92 92 92 188 188 188 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 94 94 94 193 193 193 F F F 10 200-219 94 94 94 193 193 193 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 92 92 92 187 187 187 T T T 13 260-279 92 92 92 190 190 190 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 88 88 88 177 177 177 T T T 16 320-339 94 94 94 192 192 192 F F F 17 340-359 94 94 94 192 192 192 F F F 18 360-379 91 91 91 182 182 182 F F F 19 380-399 91 91 91 182 182 182 F F F 20 400-419 91 91 91 182 182 182 F F F 21 420-439 94 94 94 193 193 193 F F F 22 440-459 93 93 93 191 191 191 F F F 23 460-479 91 91 91 182 182 182 T T T 24 480-499 92 92 92 188 188 188 T T T 25 500-519 86 86 86 175 175 175 T T T 26 520-539 90 90 90 181 181 181 T T T 27 540-559 94 94 94 193 193 193 T T T 28 560-579 90 90 90 181 181 181 T T T 29 580-599 92 92 92 186 186 186 F F F Max gen: 94, Max flush: 193 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 94 94 94 193 193 193 F F F 2 040-059 94 94 94 192 192 192 F F F 3 060-079 94 94 94 192 192 192 F F F 4 080-099 95 95 95 194 194 194 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 94 94 94 193 193 193 F F F 7 140-159 92 92 92 188 188 188 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 94 94 94 193 193 193 F F F 10 200-219 94 94 94 193 193 193 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 95 95 95 194 194 194 F F F 13 260-279 92 92 92 190 190 190 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 95 95 95 194 194 194 F F F 16 320-339 94 94 94 192 192 192 F F F 17 340-359 94 94 94 192 192 192 F F F 18 360-379 91 91 91 182 182 182 F F F 19 380-399 91 91 91 182 182 182 F F F 20 400-419 91 91 91 182 182 182 F F F 21 420-439 94 94 94 193 193 193 F F F 22 440-459 93 93 93 191 191 191 F F F 23 460-479 95 95 95 194 194 194 F F F 24 480-499 95 95 95 194 194 194 F F F 25 500-519 95 95 95 194 194 194 F F F 26 520-539 95 95 95 194 194 194 F F F 27 540-559 95 95 95 194 194 194 F F F 28 560-579 95 95 95 194 194 194 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 95, Max flush: 194 Begin loop 48 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.523140949Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.523435203Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.523453253Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566} {"msg":"Crucible b2e52d30-7e03-4e34-87ff-2f2caef7630b has session id: 6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.523889038Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Upstairs opts: Upstairs UUID: b2e52d30-7e03-4e34-87ff-2f2caef7630b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.523931199Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Crucible stats registered with UUID: b2e52d30-7e03-4e34-87ff-2f2caef7630b","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.523954229Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524290293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566} {"msg":"b2e52d30-7e03-4e34-87ff-2f2caef7630b active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524314904Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524329374Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524343814Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524357584Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524371614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524389255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524403365Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524416905Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524431045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524518146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50363)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524639128Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:45:44.52481053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51425)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.52483788Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:45:44.52485519Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:45:44.524873371Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524885961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.524899271Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.525040053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.525495529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.525522289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.525541259Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.525555359Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.526172317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.526196798Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.526675453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.526704784Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.526722174Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.526750065Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.526767895Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:44.526784395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.525360617Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46159)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.525594339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.526751684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.526797615Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.527255151Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.527922149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.52795958Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[0]R flush_numbers[0..12]: [176, 193, 192, 192, 194, 191, 193, 188, 167, 193, 193, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.52798964Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[0]R generation[0..12]: [87, 94, 94, 94, 95, 93, 94, 92, 83, 94, 94, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.52801941Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528043311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[1]R flush_numbers[0..12]: [176, 193, 192, 192, 194, 191, 193, 188, 167, 193, 193, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528061701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Wait for a query_work_queue command to finish before sending IO [1]R generation[0..12]: [87, 94, 94, 94, 95, 93, 94, 92, 83, 94, 94, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528082221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528100331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[2]R flush_numbers[0..12]: [176, 193, 192, 192, 194, 191, 193, 188, 167, 193, 193, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528117711Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[2]R generation[0..12]: [87, 94, 94, 94, 95, 93, 94, 92, 83, 94, 94, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528135392Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528153432Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528169872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528186423Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528202823Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528222803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"":"downstairs","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528239973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528255173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"0","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528272424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"1","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528289254Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"client":"2","":"downstairs","session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528306294Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_idRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528325694Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"b2e52d30-7e03-4e34-87ff-2f2caef7630b is now active with session: 6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528341854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528356905Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566,"session_id":"6d3fc6f9-2151-40ed-9af5-82d6ad1b4de1"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:45:54.528372195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 137 len 20480 data: 10 11 12 7 9 03/30 Write block 517 len 28672 data: 4 5 4 4 4 4 5 04/30 Write block 26 len 36864 data: 5 5 6 7 5 6 4 4 4 05/30 Read block 115 len 12288 06/30 Read block 179 len 16384 07/30 Read block 67 len 36864 08/30 Read block 7 len 36864 09/30 Flush 10/30 Write block 363 len 32768 data: 5 4 4 5 4 7 9 13 11/30 Read block 170 len 36864 12/30 Write block 574 len 24576 data: 6 6 9 8 10 9 13/30 Write block 378 len 12288 data: 12 10 9 14/30 Read block 68 len 16384 15/30 Read block 269 len 4096 16/30 Write block 425 len 36864 data: 8 7 8 8 8 6 6 5 5 17/30 Write block 429 len 20480 data: 9 7 7 6 6 18/30 Write block 397 len 16384 data: 9 9 10 10 19/30 Read block 42 len 4096 20/30 Write block 330 len 40960 data: 6 5 6 5 7 6 7 7 6 8 21/30 Write block 251 len 8192 data: 14 12 22/30 Read block 63 len 16384 23/30 Flush 24/30 Write block 270 len 16384 data: 4 2 3 3 25/30 Write block 209 len 24576 data: 8 9 9 8 7 6 26/30 Read block 378 len 24576 27/30 Read block 416 len 20480 28/30 Write block 189 len 28672 data: 5 4 6 8 7 8 8 29/30 Read block 578 len 40960 30/30 Write block 447 len 12288 data: 10 9 8 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Write 7 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false 31 Acked 1030 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:45:54.554056381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 96 96 96 196 196 196 F F F 2 040-059 94 94 94 192 192 192 F F F 3 060-079 94 94 94 192 192 192 F F F 4 080-099 95 95 95 194 194 194 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 96 96 96 196 196 196 F F F 7 140-159 96 96 96 196 196 196 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 94 94 94 193 193 193 T T T 10 200-219 94 94 94 193 193 193 T T T 11 220-239 92 92 92 188 188 188 F F F 12 240-259 96 96 96 197 197 197 F F F 13 260-279 92 92 92 190 190 190 T T T 14 280-299 89 89 89 179 179 179 F F F 15 300-319 95 95 95 194 194 194 F F F 16 320-339 96 96 96 197 197 197 F F F 17 340-359 94 94 94 192 192 192 F F F 18 360-379 96 96 96 197 197 197 F F F 19 380-399 96 96 96 197 197 197 F F F 20 400-419 96 96 96 197 197 197 F F F 21 420-439 96 96 96 197 197 197 F F F 22 440-459 93 93 93 191 191 191 T T T 23 460-479 95 95 95 194 194 194 F F F 24 480-499 95 95 95 194 194 194 F F F 25 500-519 96 96 96 196 196 196 F F F 26 520-539 96 96 96 196 196 196 F F F 27 540-559 95 95 95 194 194 194 F F F 28 560-579 96 96 96 197 197 197 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 96, Max flush: 197 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 96 96 96 196 196 196 F F F 2 040-059 94 94 94 192 192 192 F F F 3 060-079 94 94 94 192 192 192 F F F 4 080-099 95 95 95 194 194 194 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 96 96 96 196 196 196 F F F 7 140-159 96 96 96 196 196 196 F F F 8 160-179 83 83 83 167 167 167 F F F 9 180-199 97 97 97 198 198 198 F F F 10 200-219 97 97 97 198 198 198 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 96 96 96 197 197 197 F F F 13 260-279 97 97 97 198 198 198 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 95 95 95 194 194 194 F F F 16 320-339 96 96 96 197 197 197 F F F 17 340-359 94 94 94 192 192 192 F F F 18 360-379 96 96 96 197 197 197 F F F 19 380-399 96 96 96 197 197 197 F F F 20 400-419 96 96 96 197 197 197 F F F 21 420-439 96 96 96 197 197 197 F F F 22 440-459 97 97 97 198 198 198 F F F 23 460-479 95 95 95 194 194 194 F F F 24 480-499 95 95 95 194 194 194 F F F 25 500-519 96 96 96 196 196 196 F F F 26 520-539 96 96 96 196 196 196 F F F 27 540-559 95 95 95 194 194 194 F F F 28 560-579 96 96 96 197 197 197 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 97, Max flush: 198 Begin loop 49 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.770455977Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.770959684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.770990124Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574} {"msg":"Crucible be28fb92-d542-4e55-a3c8-805fd3f1fef4 has session id: 437e82a3-2db3-428d-a806-68f6621e35a7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771228687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Upstairs opts: Upstairs UUID: be28fb92-d542-4e55-a3c8-805fd3f1fef4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771272988Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Crucible stats registered with UUID: be28fb92-d542-4e55-a3c8-805fd3f1fef4","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771297288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771654482Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574} {"msg":"be28fb92-d542-4e55-a3c8-805fd3f1fef4 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771746854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771765444Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771780304Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771794404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771820504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771846015Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771871165Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771897876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.771960526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.772018607Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:40905)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.77220382Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:41711)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.772430802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:08.772532804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:08.772562244Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:08.772589544Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.772603945Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.772617525Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.772699306Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.772927939Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.772955659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.772970099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.772983719Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.773581627Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.773696638Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.774092124Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.774114934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.774128984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.774203465Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.774222065Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:08.774235605Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.774215915Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36072)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.774432177Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.77543912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.775467051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776027438Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776584715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776612355Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[0]R flush_numbers[0..12]: [176, 196, 192, 192, 194, 191, 196, 196, 167, 198, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776630725Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[0]R generation[0..12]: [87, 96, 94, 94, 95, 93, 96, 96, 83, 97, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776648856Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776666876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[1]R flush_numbers[0..12]: [176, 196, 192, 192, 194, 191, 196, 196, 167, 198, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776684446Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[1]R generation[0..12]: [87, 96, 94, 94, 95, 93, 96, 96, 83, 97, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776701606Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776719637Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[2]R flush_numbers[0..12]: [176, 196, 192, 192, 194, 191, 196, 196, 167, 198, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776736997Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[2]R generation[0..12]: [87, 96, 94, 94, 95, 93, 96, 96, 83, 97, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776758197Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776775907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776792247Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776808918Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776829628Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776846558Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"":"downstairs","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776863218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776878108Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"0","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776894739Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"1","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776911419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"client":"2","":"downstairs","session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776928239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.776943549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"be28fb92-d542-4e55-a3c8-805fd3f1fef4 is now active with session: 437e82a3-2db3-428d-a806-68f6621e35a7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.77695964Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.77697874Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574,"session_id":"437e82a3-2db3-428d-a806-68f6621e35a7"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:18.7769941Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 475 len 12288 data: 9 13 12 03/30 Write block 188 len 28672 data: 5 6 5 7 9 8 9 04/30 Write block 432 len 16384 data: 7 7 4 5 05/30 Read block 528 len 8192 06/30 Write block 492 len 36864 data: 7 7 8 7 8 9 9 10 11 07/30 Write block 515 len 20480 data: 3 4 5 6 5 08/30 Write block 326 len 36864 data: 7 7 6 6 7 6 7 6 8 09/30 Flush 10/30 Write block 400 len 12288 data: 11 8 7 11/30 Read block 91 len 32768 12/30 Read block 518 len 4096 13/30 Read block 83 len 32768 14/30 Read block 191 len 8192 15/30 Write block 253 len 20480 data: 11 14 16 13 10 16/30 Write block 444 len 24576 data: 10 8 8 11 10 9 17/30 Write block 335 len 16384 data: 7 8 8 7 18/30 Write block 393 len 24576 data: 9 8 8 9 10 10 19/30 Read block 253 len 32768 20/30 Read block 371 len 36864 21/30 Read block 410 len 36864 22/30 Read block 364 len 24576 23/30 Write block 164 len 16384 data: 4 5 6 7 24/30 Write block 301 len 28672 data: 4 5 6 5 5 5 6 25/30 Write block 177 len 4096 data: 6 26/30 Read block 336 len 20480 27/30 Write block 36 len 16384 data: 4 4 5 6 28/30 Read block 562 len 16384 29/30 Write block 526 len 24576 data: 12 11 11 11 11 12 30/30 Write block 87 len 28672 data: 8 8 8 8 7 7 7 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 6 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 27 28 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:18.802649845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 96 96 96 196 196 196 T T T 2 040-059 94 94 94 192 192 192 F F F 3 060-079 94 94 94 192 192 192 F F F 4 080-099 95 95 95 194 194 194 T T T 5 100-119 93 93 93 191 191 191 F F F 6 120-139 96 96 96 196 196 196 F F F 7 140-159 96 96 96 196 196 196 F F F 8 160-179 98 98 98 201 201 201 F F F 9 180-199 98 98 98 200 200 200 F F F 10 200-219 97 97 97 198 198 198 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 98 98 98 201 201 201 F F F 13 260-279 97 97 97 198 198 198 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 98 98 98 201 201 201 F F F 16 320-339 98 98 98 201 201 201 F F F 17 340-359 94 94 94 192 192 192 F F F 18 360-379 96 96 96 197 197 197 F F F 19 380-399 98 98 98 201 201 201 F F F 20 400-419 98 98 98 201 201 201 F F F 21 420-439 98 98 98 200 200 200 F F F 22 440-459 98 98 98 201 201 201 F F F 23 460-479 98 98 98 200 200 200 F F F 24 480-499 98 98 98 200 200 200 F F F 25 500-519 98 98 98 200 200 200 F F F 26 520-539 96 96 96 196 196 196 T T T 27 540-559 95 95 95 194 194 194 F F F 28 560-579 96 96 96 197 197 197 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 98, Max flush: 201 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 99 99 99 202 202 202 F F F 2 040-059 94 94 94 192 192 192 F F F 3 060-079 94 94 94 192 192 192 F F F 4 080-099 99 99 99 202 202 202 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 96 96 96 196 196 196 F F F 7 140-159 96 96 96 196 196 196 F F F 8 160-179 98 98 98 201 201 201 F F F 9 180-199 98 98 98 200 200 200 F F F 10 200-219 97 97 97 198 198 198 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 98 98 98 201 201 201 F F F 13 260-279 97 97 97 198 198 198 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 98 98 98 201 201 201 F F F 16 320-339 98 98 98 201 201 201 F F F 17 340-359 94 94 94 192 192 192 F F F 18 360-379 96 96 96 197 197 197 F F F 19 380-399 98 98 98 201 201 201 F F F 20 400-419 98 98 98 201 201 201 F F F 21 420-439 98 98 98 200 200 200 F F F 22 440-459 98 98 98 201 201 201 F F F 23 460-479 98 98 98 200 200 200 F F F 24 480-499 98 98 98 200 200 200 F F F 25 500-519 98 98 98 200 200 200 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 95 95 95 194 194 194 F F F 28 560-579 96 96 96 197 197 197 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 99, Max flush: 202 Begin loop 50 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.006865987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.007370213Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.007390973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582} {"msg":"Crucible 6c891e98-299e-4775-9b04-8d914dae9a48 has session id: b9724141-b6b7-45f7-a085-f1df3be91e66","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.007740998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Upstairs opts: Upstairs UUID: 6c891e98-299e-4775-9b04-8d914dae9a48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.007780598Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Crucible stats registered with UUID: 6c891e98-299e-4775-9b04-8d914dae9a48","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.007796958Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008120033Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582} {"msg":"6c891e98-299e-4775-9b04-8d914dae9a48 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008248334Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008265824Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008280535Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008294235Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008312075Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008327095Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008350876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008419406Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008439507Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008453937Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_connection connected from Ok(127.0.0.1:53369)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.00868164Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_connection connected from Ok(127.0.0.1:57608)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.00871801Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:33.008867742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:33.008910873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:33.008931943Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008945343Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.008959093Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.009037244Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.009606732Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.009649162Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.009678902Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.009705323Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.01029675Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.01032431Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.010858487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.010884928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.010899568Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.010912148Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.010925828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:33.010939408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.010495413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_connection connected from Ok(127.0.0.1:55182)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.010634555Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.011791759Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.01182001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.012361987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.012895223Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.012918884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[0]R flush_numbers[0..12]: [176, 202, 192, 192, 202, 191, 196, 196, 201, 200, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.012934544Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[0]R generation[0..12]: [87, 99, 94, 94, 99, 93, 96, 96, 98, 98, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.012948944Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.012963514Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[1]R flush_numbers[0..12]: [176, 202, 192, 192, 202, 191, 196, 196, 201, 200, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.012977324Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[1]R generation[0..12]: [87, 99, 94, 94, 99, 93, 96, 96, 98, 98, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.012991285Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013020785Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[2]R flush_numbers[0..12]: [176, 202, 192, 192, 202, 191, 196, 196, 201, 200, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013035585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[2]R generation[0..12]: [87, 99, 94, 94, 99, 93, 96, 96, 98, 98, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013050325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013064765Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Max found gen is 100","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2024-07-10T15:46:43.013081616Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013095226Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013111996Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013125516Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"":"downstairs","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013139206Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013151387Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"0","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013165087Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"1","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013178677Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"client":"2","":"downstairs","session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013192487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013205127Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"6c891e98-299e-4775-9b04-8d914dae9a48 is now active with session: b9724141-b6b7-45f7-a085-f1df3be91e66","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013218358Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013230368Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582,"session_id":"b9724141-b6b7-45f7-a085-f1df3be91e66"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:43.013242818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 164 len 28672 data: 5 6 7 8 7 5 5 02/30 Read block 67 len 20480 03/30 Read block 407 len 8192 04/30 Read block 310 len 32768 05/30 Write block 430 len 12288 data: 8 8 8 06/30 Write block 331 len 8192 data: 7 8 07/30 Write block 343 len 36864 data: 9 10 9 8 10 10 7 7 8 08/30 Read block 44 len 4096 09/30 Read block 426 len 20480 10/30 Write block 331 len 16384 data: 8 9 7 9 11/30 Write block 317 len 40960 data: 7 6 8 8 8 9 8 8 8 8 12/30 Read block 572 len 8192 13/30 Write block 256 len 28672 data: 14 11 8 9 11 9 8 14/30 Flush 15/30 Write block 399 len 40960 data: 11 12 9 8 8 7 7 8 8 6 16/30 Read block 583 len 12288 17/30 Read block 74 len 24576 18/30 Read block 462 len 12288 19/30 Read block 204 len 8192 20/30 Read block 409 len 12288 21/30 Read block 169 len 20480 22/30 Read block 231 len 16384 23/30 Write block 59 len 32768 data: 8 8 9 5 5 7 10 8 24/30 Write block 361 len 24576 data: 9 4 6 5 5 6 25/30 Read block 48 len 8192 26/30 Flush 27/30 Write block 28 len 12288 data: 7 8 6 28/30 Read block 504 len 4096 29/30 Read block 337 len 4096 30/30 Read block 297 len 4096 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 1 Done Done Done false 31 Acked 1030 Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:43.041261253Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 99 99 99 202 202 202 T T T 2 040-059 100 100 100 205 205 205 F F F 3 060-079 100 100 100 205 205 205 F F F 4 080-099 99 99 99 202 202 202 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 96 96 96 196 196 196 F F F 7 140-159 96 96 96 196 196 196 F F F 8 160-179 100 100 100 203 203 203 F F F 9 180-199 98 98 98 200 200 200 F F F 10 200-219 97 97 97 198 198 198 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 100 100 100 203 203 203 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 100 100 100 203 203 203 F F F 16 320-339 100 100 100 203 203 203 F F F 17 340-359 100 100 100 203 203 203 F F F 18 360-379 100 100 100 205 205 205 F F F 19 380-399 100 100 100 204 204 204 F F F 20 400-419 100 100 100 204 204 204 F F F 21 420-439 100 100 100 203 203 203 F F F 22 440-459 98 98 98 201 201 201 F F F 23 460-479 98 98 98 200 200 200 F F F 24 480-499 98 98 98 200 200 200 F F F 25 500-519 98 98 98 200 200 200 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 95 95 95 194 194 194 F F F 28 560-579 96 96 96 197 197 197 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 100, Max flush: 205 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 101 101 101 206 206 206 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 100 100 100 205 205 205 F F F 4 080-099 99 99 99 202 202 202 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 96 96 96 196 196 196 F F F 7 140-159 96 96 96 196 196 196 F F F 8 160-179 100 100 100 203 203 203 F F F 9 180-199 98 98 98 200 200 200 F F F 10 200-219 97 97 97 198 198 198 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 100 100 100 203 203 203 F F F 14 280-299 89 89 89 179 179 179 F F F 15 300-319 100 100 100 203 203 203 F F F 16 320-339 100 100 100 203 203 203 F F F 17 340-359 100 100 100 203 203 203 F F F 18 360-379 100 100 100 205 205 205 F F F 19 380-399 100 100 100 204 204 204 F F F 20 400-419 100 100 100 204 204 204 F F F 21 420-439 100 100 100 203 203 203 F F F 22 440-459 98 98 98 201 201 201 F F F 23 460-479 98 98 98 200 200 200 F F F 24 480-499 98 98 98 200 200 200 F F F 25 500-519 98 98 98 200 200 200 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 95 95 95 194 194 194 F F F 28 560-579 96 96 96 197 197 197 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 101, Max flush: 206 Begin loop 51 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.215866369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.216146113Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.216162683Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590} {"msg":"Crucible 70849a47-dd1a-43cd-b8f2-5f541a336ddb has session id: 2c9e61c6-90f9-4166-929b-ad97a572c038","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.216594178Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Upstairs opts: Upstairs UUID: 70849a47-dd1a-43cd-b8f2-5f541a336ddb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.216639499Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Crucible stats registered with UUID: 70849a47-dd1a-43cd-b8f2-5f541a336ddb","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.216662099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.216985313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590} {"msg":"70849a47-dd1a-43cd-b8f2-5f541a336ddb active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217032224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217054804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217077165Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217099085Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217120595Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217146205Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217168696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217190136Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217212646Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217235107Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:53.21746753Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:53.21750948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_connection connected from Ok(127.0.0.1:62631)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.21753995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_connection connected from Ok(127.0.0.1:53193)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217566311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:46:53.217593291Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217615551Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217631652Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.217645542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.218108818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.218134908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.218149708Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.218163208Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.218732715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.218865477Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.219221122Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.219249802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.219267642Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.219385424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.219407214Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:46:53.219424084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.218492092Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_connection connected from Ok(127.0.0.1:32789)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.218829347Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.220062912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.220096663Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.220565949Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221102116Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221121386Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[0]R flush_numbers[0..12]: [176, 206, 205, 205, 202, 191, 196, 196, 203, 200, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221136096Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[0]R generation[0..12]: [87, 101, 100, 100, 99, 93, 96, 96, 100, 98, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221150406Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221164866Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[1]R flush_numbers[0..12]: [176, 206, 205, 205, 202, 191, 196, 196, 203, 200, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221178807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[1]R generation[0..12]: [87, 101, 100, 100, 99, 93, 96, 96, 100, 98, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221192637Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221206797Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[2]R flush_numbers[0..12]: [176, 206, 205, 205, 202, 191, 196, 196, 203, 200, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221220867Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[2]R generation[0..12]: [87, 101, 100, 100, 99, 93, 96, 96, 100, 98, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221234787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221249467Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221262768Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221279228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221295858Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221309458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"":"downstairs","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221322908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221335108Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"0","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221348549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"1","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221361839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"client":"2","":"downstairs","session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221375289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221387729Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"70849a47-dd1a-43cd-b8f2-5f541a336ddb is now active with session: 2c9e61c6-90f9-4166-929b-ad97a572c038","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.221400769Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.22141288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590,"session_id":"2c9e61c6-90f9-4166-929b-ad97a572c038"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:03.22142528Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 439 len 16384 data: 9 9 14 12 03/30 Flush 04/30 Write block 308 len 16384 data: 5 4 4 3 05/30 Read block 95 len 28672 06/30 Read block 512 len 16384 07/30 Write block 385 len 40960 data: 7 10 7 7 7 8 8 8 10 9 08/30 Read block 443 len 8192 09/30 Read block 129 len 4096 10/30 Read block 348 len 36864 11/30 Write block 322 len 12288 data: 10 9 9 12/30 Read block 106 len 12288 13/30 Flush 14/30 Write block 73 len 36864 data: 7 7 7 6 7 7 7 6 5 15/30 Read block 204 len 16384 16/30 Write block 570 len 36864 data: 6 5 5 5 7 7 10 9 11 17/30 Read block 66 len 12288 18/30 Write block 540 len 4096 data: 8 19/30 Read block 499 len 36864 20/30 Write block 130 len 40960 data: 12 11 9 7 9 9 8 11 12 13 21/30 Read block 383 len 32768 22/30 Flush 23/30 Write block 475 len 32768 data: 10 14 13 13 11 10 10 9 24/30 Read block 293 len 20480 25/30 Write block 173 len 24576 data: 7 8 6 4 7 6 26/30 Write block 286 len 32768 data: 6 6 6 6 5 6 7 7 27/30 Read block 246 len 20480 28/30 Write block 82 len 4096 data: 5 29/30 Read block 175 len 16384 30/30 Read block 275 len 20480 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Read 4 Done Done Done false 31 Acked 1030 Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 8 7 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:47:03.249118181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 101 101 101 206 206 206 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 102 102 102 210 210 210 F F F 4 080-099 102 102 102 210 210 210 T T T 5 100-119 93 93 93 191 191 191 F F F 6 120-139 102 102 102 210 210 210 F F F 7 140-159 96 96 96 196 196 196 F F F 8 160-179 100 100 100 203 203 203 T T T 9 180-199 98 98 98 200 200 200 F F F 10 200-219 97 97 97 198 198 198 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 100 100 100 203 203 203 F F F 14 280-299 89 89 89 179 179 179 T T T 15 300-319 102 102 102 209 209 209 F F F 16 320-339 102 102 102 209 209 209 F F F 17 340-359 100 100 100 203 203 203 F F F 18 360-379 100 100 100 205 205 205 F F F 19 380-399 102 102 102 209 209 209 F F F 20 400-419 100 100 100 204 204 204 F F F 21 420-439 102 102 102 208 208 208 F F F 22 440-459 102 102 102 208 208 208 F F F 23 460-479 98 98 98 200 200 200 T T T 24 480-499 98 98 98 200 200 200 T T T 25 500-519 98 98 98 200 200 200 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 102 102 102 210 210 210 F F F 28 560-579 102 102 102 210 210 210 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 102, Max flush: 210 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 101 101 101 206 206 206 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 102 102 102 210 210 210 F F F 4 080-099 103 103 103 211 211 211 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 102 102 102 210 210 210 F F F 7 140-159 96 96 96 196 196 196 F F F 8 160-179 103 103 103 211 211 211 F F F 9 180-199 98 98 98 200 200 200 F F F 10 200-219 97 97 97 198 198 198 F F F 11 220-239 92 92 92 188 188 188 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 100 100 100 203 203 203 F F F 14 280-299 103 103 103 211 211 211 F F F 15 300-319 102 102 102 209 209 209 F F F 16 320-339 102 102 102 209 209 209 F F F 17 340-359 100 100 100 203 203 203 F F F 18 360-379 100 100 100 205 205 205 F F F 19 380-399 102 102 102 209 209 209 F F F 20 400-419 100 100 100 204 204 204 F F F 21 420-439 102 102 102 208 208 208 F F F 22 440-459 102 102 102 208 208 208 F F F 23 460-479 103 103 103 211 211 211 F F F 24 480-499 103 103 103 211 211 211 F F F 25 500-519 98 98 98 200 200 200 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 102 102 102 210 210 210 F F F 28 560-579 102 102 102 210 210 210 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 103, Max flush: 211 Begin loop 52 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.479750708Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.480301225Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.480329246Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598} {"msg":"Crucible 012fa31c-64da-485c-8231-73892e933a9f has session id: f45616b9-d6de-4daa-96a2-1faf5604890a","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.480549378Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Upstairs opts: Upstairs UUID: 012fa31c-64da-485c-8231-73892e933a9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.480582899Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Crucible stats registered with UUID: 012fa31c-64da-485c-8231-73892e933a9f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.480598209Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.480896633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598} {"msg":"012fa31c-64da-485c-8231-73892e933a9f active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.480969894Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.480997014Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481022854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481038435Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481056455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481070365Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481084635Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481106405Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481121645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481135506Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61245)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.48145618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53674)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481635782Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:47:17.481752233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:47:17.481777654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:47:17.481804724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481829674Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481868985Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.481968906Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.482163339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.48224196Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.48227797Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.482306091Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.482690535Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.482797877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.483178302Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.483216312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.483244952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.483274133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.483302183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:17.483323744Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.482694015Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63703)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.482821297Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.484073083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.484123674Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.4846186Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485144907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485168887Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[0]R flush_numbers[0..12]: [176, 206, 205, 210, 211, 191, 210, 196, 211, 200, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485184497Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[0]R generation[0..12]: [87, 101, 100, 102, 103, 93, 102, 96, 103, 98, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485199047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485213377Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[1]R flush_numbers[0..12]: [176, 206, 205, 210, 211, 191, 210, 196, 211, 200, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485227157Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[1]R generation[0..12]: [87, 101, 100, 102, 103, 93, 102, 96, 103, 98, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485240898Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485255008Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[2]R flush_numbers[0..12]: [176, 206, 205, 210, 211, 191, 210, 196, 211, 200, 198, 188]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485268888Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[2]R generation[0..12]: [87, 101, 100, 102, 103, 93, 102, 96, 103, 98, 97, 92]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485282888Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485296918Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:47:27.485310018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485326679Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485343619Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485357509Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"":"downstairs","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485371159Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485383499Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"0","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.48539685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"1","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.4854101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"client":"2","":"downstairs","session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.48542358Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.48543585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"012fa31c-64da-485c-8231-73892e933a9f is now active with session: f45616b9-d6de-4daa-96a2-1faf5604890a","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.48544876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485460851Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598,"session_id":"f45616b9-d6de-4daa-96a2-1faf5604890a"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:27.485473301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 155 len 32768 data: 6 7 6 9 8 5 4 4 03/30 Read block 581 len 12288 04/30 Read block 263 len 20480 05/30 Write block 287 len 12288 data: 7 7 7 06/30 Flush 07/30 Write block 395 len 36864 data: 9 10 11 11 12 13 10 9 9 08/30 Read block 575 len 24576 09/30 Write block 144 len 16384 data: 10 8 8 10 10/30 Read block 424 len 20480 11/30 Read block 400 len 12288 12/30 Read block 73 len 20480 13/30 Read block 594 len 8192 14/30 Write block 143 len 36864 data: 11 11 9 9 11 11 11 8 8 15/30 Read block 550 len 32768 16/30 Write block 235 len 16384 data: 13 13 11 9 17/30 Write block 196 len 24576 data: 9 11 9 7 7 6 18/30 Read block 402 len 40960 19/30 Read block 347 len 12288 20/30 Write block 124 len 12288 data: 6 6 5 21/30 Read block 105 len 4096 22/30 Read block 175 len 8192 23/30 Read block 53 len 32768 24/30 Read block 561 len 36864 25/30 Write block 20 len 8192 data: 8 8 26/30 Flush 27/30 Write block 62 len 20480 data: 6 6 8 11 9 28/30 Write block 338 len 40960 data: 8 9 9 10 10 10 11 10 9 11 29/30 Read block 474 len 24576 30/30 Write block 558 len 12288 data: 7 4 3 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 10 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:47:27.513679358Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 104 104 104 215 215 215 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 102 102 102 210 210 210 T T T 4 080-099 103 103 103 211 211 211 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 104 104 104 214 214 214 F F F 7 140-159 104 104 104 214 214 214 F F F 8 160-179 104 104 104 213 213 213 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 104 104 104 214 214 214 F F F 11 220-239 104 104 104 214 214 214 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 100 100 100 203 203 203 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 102 102 102 209 209 209 F F F 16 320-339 102 102 102 209 209 209 T T T 17 340-359 100 100 100 203 203 203 T T T 18 360-379 100 100 100 205 205 205 F F F 19 380-399 104 104 104 214 214 214 F F F 20 400-419 104 104 104 214 214 214 F F F 21 420-439 102 102 102 208 208 208 F F F 22 440-459 102 102 102 208 208 208 F F F 23 460-479 103 103 103 211 211 211 F F F 24 480-499 103 103 103 211 211 211 F F F 25 500-519 98 98 98 200 200 200 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 102 102 102 210 210 210 T T T 28 560-579 102 102 102 210 210 210 T T T 29 580-599 92 92 92 186 186 186 F F F Max gen: 104, Max flush: 215 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 104 104 104 215 215 215 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 105 105 105 216 216 216 F F F 4 080-099 103 103 103 211 211 211 F F F 5 100-119 93 93 93 191 191 191 F F F 6 120-139 104 104 104 214 214 214 F F F 7 140-159 104 104 104 214 214 214 F F F 8 160-179 104 104 104 213 213 213 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 104 104 104 214 214 214 F F F 11 220-239 104 104 104 214 214 214 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 100 100 100 203 203 203 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 102 102 102 209 209 209 F F F 16 320-339 105 105 105 216 216 216 F F F 17 340-359 105 105 105 216 216 216 F F F 18 360-379 100 100 100 205 205 205 F F F 19 380-399 104 104 104 214 214 214 F F F 20 400-419 104 104 104 214 214 214 F F F 21 420-439 102 102 102 208 208 208 F F F 22 440-459 102 102 102 208 208 208 F F F 23 460-479 103 103 103 211 211 211 F F F 24 480-499 103 103 103 211 211 211 F F F 25 500-519 98 98 98 200 200 200 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 105 105 105 216 216 216 F F F 28 560-579 105 105 105 216 216 216 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 105, Max flush: 216 Begin loop 53 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.749844155Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.750172139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.75020042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606} {"msg":"Crucible 93534553-b665-4742-b88f-ab12cffd3c6e has session id: 6e294628-f301-48e0-a52c-2c557a2a39bf","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.750691746Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Upstairs opts: Upstairs UUID: 93534553-b665-4742-b88f-ab12cffd3c6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.750738767Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Crucible stats registered with UUID: 93534553-b665-4742-b88f-ab12cffd3c6e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.750759907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751100241Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606} {"msg":"93534553-b665-4742-b88f-ab12cffd3c6e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751200893Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751220263Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751235173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751249313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751271433Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751294904Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751319794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751342985Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751365615Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751384915Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:41844)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751656218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:47:41.75177331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:47:41.75179605Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:47:41.75181265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.7518255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751839201Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:60776)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751860951Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.751953732Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.752480159Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.752511149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.752526369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.75254228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.753118327Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.753146217Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.753660224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.753680184Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.753694054Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.753757175Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.753775715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:41.753789516Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.752302066Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:40561)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.752460629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.753697244Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.753725045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.754291322Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.754831579Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.754851829Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[0]R flush_numbers[0..12]: [176, 215, 205, 216, 211, 191, 214, 214, 213, 214, 214, 214]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.754866599Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[0]R generation[0..12]: [87, 104, 100, 105, 103, 93, 104, 104, 104, 104, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.754880849Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.754895219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[1]R flush_numbers[0..12]: [176, 215, 205, 216, 211, 191, 214, 214, 213, 214, 214, 214]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.75490933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[1]R generation[0..12]: [87, 104, 100, 105, 103, 93, 104, 104, 104, 104, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.75492341Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.75493756Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[2]R flush_numbers[0..12]: [176, 215, 205, 216, 211, 191, 214, 214, 213, 214, 214, 214]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.75495158Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[2]R generation[0..12]: [87, 104, 100, 105, 103, 93, 104, 104, 104, 104, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.75496564Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.754979791Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.754992941Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755006211Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755025741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755039741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"":"downstairs","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755053221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755065342Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"0","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755078762Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"1","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755100032Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"client":"2","":"downstairs","session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755124462Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755146703Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"93534553-b665-4742-b88f-ab12cffd3c6e is now active with session: 6e294628-f301-48e0-a52c-2c557a2a39bf","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755170923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755193103Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606,"session_id":"6e294628-f301-48e0-a52c-2c557a2a39bf"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:47:51.755212114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 100 len 20480 data: 7 6 5 6 5 02/30 Write block 346 len 32768 data: 10 12 11 8 8 9 9 10 03/30 Read block 573 len 28672 04/30 Read block 520 len 24576 05/30 Write block 70 len 36864 data: 8 7 6 8 8 8 7 8 8 06/30 Read block 39 len 40960 07/30 Read block 64 len 24576 08/30 Read block 84 len 16384 09/30 Read block 476 len 40960 10/30 Read block 363 len 20480 11/30 Read block 552 len 4096 12/30 Read block 54 len 28672 13/30 Read block 501 len 4096 14/30 Write block 156 len 32768 data: 8 7 10 9 6 5 5 4 15/30 Read block 136 len 36864 16/30 Read block 196 len 36864 17/30 Write block 68 len 24576 data: 8 8 9 8 7 9 18/30 Write block 234 len 4096 data: 12 19/30 Read block 511 len 32768 20/30 Read block 180 len 28672 21/30 Read block 182 len 24576 22/30 Write block 342 len 36864 data: 11 11 12 11 11 13 12 9 9 23/30 Write block 112 len 8192 data: 6 8 24/30 Read block 396 len 28672 25/30 Read block 283 len 40960 26/30 Flush 27/30 Write block 323 len 28672 data: 10 10 9 9 8 7 7 28/30 Write block 408 len 12288 data: 7 6 5 29/30 Write block 205 len 32768 data: 5 5 6 7 9 10 10 9 30/30 Write block 308 len 20480 data: 6 5 5 4 3 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Write 8 Done Done Done false 31 Acked 1030 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:47:51.781075371Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 104 104 104 215 215 215 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 106 106 106 217 217 217 F F F 4 080-099 103 103 103 211 211 211 F F F 5 100-119 106 106 106 218 218 218 F F F 6 120-139 104 104 104 214 214 214 F F F 7 140-159 106 106 106 217 217 217 F F F 8 160-179 106 106 106 217 217 217 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 104 104 104 214 214 214 T T T 11 220-239 106 106 106 217 217 217 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 100 100 100 203 203 203 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 102 102 102 209 209 209 T T T 16 320-339 105 105 105 216 216 216 T T T 17 340-359 106 106 106 217 217 217 F F F 18 360-379 100 100 100 205 205 205 F F F 19 380-399 104 104 104 214 214 214 F F F 20 400-419 104 104 104 214 214 214 T T T 21 420-439 102 102 102 208 208 208 F F F 22 440-459 102 102 102 208 208 208 F F F 23 460-479 103 103 103 211 211 211 F F F 24 480-499 103 103 103 211 211 211 F F F 25 500-519 98 98 98 200 200 200 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 105 105 105 216 216 216 F F F 28 560-579 105 105 105 216 216 216 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 106, Max flush: 218 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 104 104 104 215 215 215 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 106 106 106 217 217 217 F F F 4 080-099 103 103 103 211 211 211 F F F 5 100-119 106 106 106 218 218 218 F F F 6 120-139 104 104 104 214 214 214 F F F 7 140-159 106 106 106 217 217 217 F F F 8 160-179 106 106 106 217 217 217 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 107 107 107 219 219 219 F F F 11 220-239 106 106 106 217 217 217 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 100 100 100 203 203 203 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 107 107 107 219 219 219 F F F 16 320-339 107 107 107 219 219 219 F F F 17 340-359 106 106 106 217 217 217 F F F 18 360-379 100 100 100 205 205 205 F F F 19 380-399 104 104 104 214 214 214 F F F 20 400-419 107 107 107 219 219 219 F F F 21 420-439 102 102 102 208 208 208 F F F 22 440-459 102 102 102 208 208 208 F F F 23 460-479 103 103 103 211 211 211 F F F 24 480-499 103 103 103 211 211 211 F F F 25 500-519 98 98 98 200 200 200 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 105 105 105 216 216 216 F F F 28 560-579 105 105 105 216 216 216 F F F 29 580-599 92 92 92 186 186 186 F F F Max gen: 107, Max flush: 219 Begin loop 54 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.99526228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.995569164Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.995638324Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614} {"msg":"Crucible fdb89263-2948-4c8f-9640-c76dbf0818d8 has session id: 5b5a5132-cde8-4238-932a-59da0f84e3e7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.995798936Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Upstairs opts: Upstairs UUID: fdb89263-2948-4c8f-9640-c76dbf0818d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.995938158Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Crucible stats registered with UUID: fdb89263-2948-4c8f-9640-c76dbf0818d8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.995971049Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996415924Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614} {"msg":"fdb89263-2948-4c8f-9640-c76dbf0818d8 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996441055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996455625Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996470145Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996484155Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996501575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996515356Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996529206Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996577916Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996616917Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996644817Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45148)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996791279Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:05.9968334Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:05.99686605Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:05.99689885Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996923361Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996947531Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63674)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.996970181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.997045912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.997607179Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.99763797Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.99765357Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.997724891Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.998311078Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.998339489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.998877475Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.998903406Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.998918776Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.998931616Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.998945356Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:05.998958847Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:15.998493541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34533)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:15.998684643Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.00001758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.000332364Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.000945452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001487928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001511619Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[0]R flush_numbers[0..12]: [176, 215, 205, 217, 211, 218, 214, 217, 217, 214, 219, 217]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001527389Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[0]R generation[0..12]: [87, 104, 100, 106, 103, 106, 104, 106, 106, 104, 107, 106]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001542139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001556569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[1]R flush_numbers[0..12]: [176, 215, 205, 217, 211, 218, 214, 217, 217, 214, 219, 217]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001570689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[1]R generation[0..12]: [87, 104, 100, 106, 103, 106, 104, 106, 106, 104, 107, 106]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.00158512Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.00159954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[2]R flush_numbers[0..12]: [176, 215, 205, 217, 211, 218, 214, 217, 217, 214, 219, 217]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.0016136Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[2]R generation[0..12]: [87, 104, 100, 106, 103, 106, 104, 106, 106, 104, 107, 106]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.00163583Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:48:16.001660141Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001683121Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001705671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001733622Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001758192Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"":"downstairs","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001775692Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001788072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"0","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001810112Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"1","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001834223Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"client":"2","":"downstairs","session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001858433Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001881364Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"fdb89263-2948-4c8f-9640-c76dbf0818d8 is now active with session: 5b5a5132-cde8-4238-932a-59da0f84e3e7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001903254Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001916474Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614,"session_id":"5b5a5132-cde8-4238-932a-59da0f84e3e7"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:16.001929024Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 132 len 36864 data: 10 8 10 10 9 12 13 14 8 02/30 Read block 575 len 4096 03/30 Flush 04/30 Write block 594 len 20480 data: 7 6 5 4 4 05/30 Write block 22 len 32768 data: 7 5 5 6 6 6 8 9 06/30 Write block 96 len 24576 data: 7 7 6 6 8 7 07/30 Write block 259 len 36864 data: 10 12 10 9 7 6 6 6 4 08/30 Write block 326 len 40960 data: 10 9 8 8 8 9 10 8 10 8 09/30 Read block 192 len 40960 10/30 Read block 403 len 24576 11/30 Write block 146 len 24576 data: 10 12 12 12 9 9 12/30 Read block 408 len 8192 13/30 Write block 31 len 32768 data: 7 5 5 5 5 5 5 6 14/30 Read block 64 len 28672 15/30 Read block 420 len 28672 16/30 Write block 98 len 20480 data: 7 7 9 8 6 17/30 Write block 509 len 36864 data: 7 6 6 4 3 4 4 5 6 18/30 Write block 418 len 4096 data: 6 19/30 Write block 269 len 20480 data: 2 5 3 4 4 20/30 Read block 86 len 12288 21/30 Read block 264 len 4096 22/30 Write block 484 len 40960 data: 8 7 6 6 6 6 7 8 8 8 23/30 Read block 135 len 8192 24/30 Write block 451 len 24576 data: 10 11 10 7 7 6 25/30 Write block 196 len 36864 data: 10 12 10 8 8 7 5 6 6 26/30 Write block 303 len 32768 data: 7 6 6 6 7 7 6 6 27/30 Read block 565 len 36864 28/30 Write block 154 len 36864 data: 7 7 9 8 11 10 7 6 6 29/30 Read block 38 len 40960 30/30 Read block 516 len 12288 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 1 Done Done Done false 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 9 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false 31 Acked 1030 Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 13 12 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:16.030366955Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 108 108 108 221 221 221 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 106 106 106 217 217 217 F F F 4 080-099 108 108 108 221 221 221 F F F 5 100-119 108 108 108 221 221 221 F F F 6 120-139 108 108 108 220 220 220 F F F 7 140-159 108 108 108 221 221 221 T T T 8 160-179 106 106 106 217 217 217 T T T 9 180-199 104 104 104 214 214 214 T T T 10 200-219 107 107 107 219 219 219 T T T 11 220-239 106 106 106 217 217 217 F F F 12 240-259 108 108 108 221 221 221 F F F 13 260-279 108 108 108 221 221 221 T T T 14 280-299 104 104 104 213 213 213 F F F 15 300-319 107 107 107 219 219 219 T T T 16 320-339 108 108 108 221 221 221 F F F 17 340-359 106 106 106 217 217 217 F F F 18 360-379 100 100 100 205 205 205 F F F 19 380-399 104 104 104 214 214 214 F F F 20 400-419 107 107 107 219 219 219 T T T 21 420-439 102 102 102 208 208 208 F F F 22 440-459 102 102 102 208 208 208 T T T 23 460-479 103 103 103 211 211 211 F F F 24 480-499 103 103 103 211 211 211 T T T 25 500-519 108 108 108 221 221 221 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 105 105 105 216 216 216 F F F 28 560-579 105 105 105 216 216 216 F F F 29 580-599 108 108 108 221 221 221 F F F Max gen: 108, Max flush: 221 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 108 108 108 221 221 221 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 106 106 106 217 217 217 F F F 4 080-099 108 108 108 221 221 221 F F F 5 100-119 108 108 108 221 221 221 F F F 6 120-139 108 108 108 220 220 220 F F F 7 140-159 109 109 109 222 222 222 F F F 8 160-179 109 109 109 222 222 222 F F F 9 180-199 109 109 109 222 222 222 F F F 10 200-219 109 109 109 222 222 222 F F F 11 220-239 106 106 106 217 217 217 F F F 12 240-259 108 108 108 221 221 221 F F F 13 260-279 109 109 109 222 222 222 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 109 109 109 222 222 222 F F F 16 320-339 108 108 108 221 221 221 F F F 17 340-359 106 106 106 217 217 217 F F F 18 360-379 100 100 100 205 205 205 F F F 19 380-399 104 104 104 214 214 214 F F F 20 400-419 109 109 109 222 222 222 F F F 21 420-439 102 102 102 208 208 208 F F F 22 440-459 109 109 109 222 222 222 F F F 23 460-479 103 103 103 211 211 211 F F F 24 480-499 109 109 109 222 222 222 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 105 105 105 216 216 216 F F F 28 560-579 105 105 105 216 216 216 F F F 29 580-599 108 108 108 221 221 221 F F F Max gen: 109, Max flush: 222 Begin loop 55 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.313292315Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.313597789Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.313617749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622} {"msg":"Crucible 9323bb97-554f-4d6f-9170-68be660d60f1 has session id: 18886916-bb61-42df-8e4c-537c0637c0ef","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314107026Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Upstairs opts: Upstairs UUID: 9323bb97-554f-4d6f-9170-68be660d60f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314152706Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Crucible stats registered with UUID: 9323bb97-554f-4d6f-9170-68be660d60f1","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314176156Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.31447066Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622} {"msg":"9323bb97-554f-4d6f-9170-68be660d60f1 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314493291Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314510351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314525301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314539171Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314556232Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314579872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314598222Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314612062Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314626072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314639813Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:60645)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314872765Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:34801)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.314913736Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:30.315110739Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:30.315179559Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:30.31520675Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.31522097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.31524294Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.315293621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.315576174Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.315603625Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.315619325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.315633365Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.316228753Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.316256493Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.31682725Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.31685143Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.316866151Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.316879561Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.316893231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:30.316906631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.316501026Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:56669)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.316725209Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.317557129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.31759225Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318104216Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318622453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318645123Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[0]R flush_numbers[0..12]: [176, 221, 205, 217, 221, 221, 220, 222, 222, 222, 222, 217]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318663404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[0]R generation[0..12]: [87, 108, 100, 106, 108, 108, 108, 109, 109, 109, 109, 106]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318681254Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318699344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[1]R flush_numbers[0..12]: [176, 221, 205, 217, 221, 221, 220, 222, 222, 222, 222, 217]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318717204Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[1]R generation[0..12]: [87, 108, 100, 106, 108, 108, 108, 109, 109, 109, 109, 106]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318734534Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318752134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[2]R flush_numbers[0..12]: [176, 221, 205, 217, 221, 221, 220, 222, 222, 222, 222, 217]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318769765Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[2]R generation[0..12]: [87, 108, 100, 106, 108, 108, 108, 109, 109, 109, 109, 106]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-07-10T15:48:40.318786865Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318814215Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318831406Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318848156Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318864586Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318880956Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"":"downstairs","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318897666Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318912696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"0","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318929537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"1","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318946167Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"client":"2","":"downstairs","session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318962997Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318978357Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"9323bb97-554f-4d6f-9170-68be660d60f1 is now active with session: 18886916-bb61-42df-8e4c-537c0637c0ef","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.318994708Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.319009788Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622,"session_id":"18886916-bb61-42df-8e4c-537c0637c0ef"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:40.319025108Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 585 len 16384 data: 6 8 7 8 02/30 Read block 590 len 36864 03/30 Write block 378 len 20480 data: 13 11 10 8 8 04/30 Read block 98 len 40960 05/30 Read block 180 len 24576 06/30 Read block 431 len 16384 07/30 Read block 68 len 40960 08/30 Read block 496 len 24576 09/30 Read block 512 len 40960 10/30 Write block 193 len 20480 data: 9 10 9 11 13 11/30 Read block 367 len 24576 12/30 Flush 13/30 Write block 368 len 8192 data: 8 10 14/30 Read block 328 len 8192 15/30 Read block 40 len 4096 16/30 Write block 47 len 24576 data: 7 6 7 7 9 12 17/30 Read block 555 len 32768 18/30 Read block 236 len 8192 19/30 Write block 560 len 12288 data: 4 4 6 20/30 Read block 210 len 20480 21/30 Write block 70 len 16384 data: 10 9 8 10 22/30 Read block 544 len 40960 23/30 Read block 307 len 28672 24/30 Read block 422 len 8192 25/30 Read block 466 len 28672 26/30 Read block 15 len 8192 27/30 Write block 216 len 28672 data: 6 5 5 7 7 10 11 28/30 Write block 312 len 12288 data: 4 4 5 29/30 Read block 410 len 28672 30/30 Read block 512 len 16384 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 7 Done Done Done false 31 Acked 1030 Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:40.344575822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 108 108 108 221 221 221 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 106 106 106 217 217 217 T T T 4 080-099 108 108 108 221 221 221 F F F 5 100-119 108 108 108 221 221 221 F F F 6 120-139 108 108 108 220 220 220 F F F 7 140-159 109 109 109 222 222 222 F F F 8 160-179 109 109 109 222 222 222 F F F 9 180-199 110 110 110 223 223 223 F F F 10 200-219 109 109 109 222 222 222 T T T 11 220-239 106 106 106 217 217 217 T T T 12 240-259 108 108 108 221 221 221 F F F 13 260-279 109 109 109 222 222 222 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 109 109 109 222 222 222 T T T 16 320-339 108 108 108 221 221 221 F F F 17 340-359 106 106 106 217 217 217 F F F 18 360-379 110 110 110 224 224 224 F F F 19 380-399 110 110 110 223 223 223 F F F 20 400-419 109 109 109 222 222 222 F F F 21 420-439 102 102 102 208 208 208 F F F 22 440-459 109 109 109 222 222 222 F F F 23 460-479 103 103 103 211 211 211 F F F 24 480-499 109 109 109 222 222 222 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 105 105 105 216 216 216 F F F 28 560-579 110 110 110 224 224 224 F F F 29 580-599 110 110 110 223 223 223 F F F Max gen: 110, Max flush: 224 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 F F F 1 020-039 108 108 108 221 221 221 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 111 111 111 225 225 225 F F F 4 080-099 108 108 108 221 221 221 F F F 5 100-119 108 108 108 221 221 221 F F F 6 120-139 108 108 108 220 220 220 F F F 7 140-159 109 109 109 222 222 222 F F F 8 160-179 109 109 109 222 222 222 F F F 9 180-199 110 110 110 223 223 223 F F F 10 200-219 111 111 111 225 225 225 F F F 11 220-239 111 111 111 225 225 225 F F F 12 240-259 108 108 108 221 221 221 F F F 13 260-279 109 109 109 222 222 222 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 111 111 111 225 225 225 F F F 16 320-339 108 108 108 221 221 221 F F F 17 340-359 106 106 106 217 217 217 F F F 18 360-379 110 110 110 224 224 224 F F F 19 380-399 110 110 110 223 223 223 F F F 20 400-419 109 109 109 222 222 222 F F F 21 420-439 102 102 102 208 208 208 F F F 22 440-459 109 109 109 222 222 222 F F F 23 460-479 103 103 103 211 211 211 F F F 24 480-499 109 109 109 222 222 222 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 105 105 105 216 216 216 F F F 28 560-579 110 110 110 224 224 224 F F F 29 580-599 110 110 110 223 223 223 F F F Max gen: 111, Max flush: 225 Begin loop 56 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.562454127Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.562953594Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.562982094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630} {"msg":"Crucible 0eb2f793-7a93-4c7c-a6d1-65a0f3bfd0c7 has session id: 2bd287d9-c922-46eb-baee-87622c5d5f1e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563063165Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Upstairs opts: Upstairs UUID: 0eb2f793-7a93-4c7c-a6d1-65a0f3bfd0c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563229717Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Crucible stats registered with UUID: 0eb2f793-7a93-4c7c-a6d1-65a0f3bfd0c7","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563264468Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563553671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630} {"msg":"0eb2f793-7a93-4c7c-a6d1-65a0f3bfd0c7 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563583082Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563602192Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563617282Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563631052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563649302Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563673393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563699353Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563724243Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563750724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.563775984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:54.564123888Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:54.564164649Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:48:54.564190709Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.564204329Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.5642177Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.56423483Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:32829)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.56424996Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45266)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.56426435Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.564912589Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.564939399Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.564954439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.5650105Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.565593967Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.565621158Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.566109934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.566136074Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.566150594Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.566221305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.566240135Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:48:54.566253815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.564331582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61194)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.564451843Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.565472696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.565501076Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566026543Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566629091Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566654831Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[0]R flush_numbers[0..12]: [176, 221, 224, 225, 221, 221, 220, 222, 222, 223, 225, 225]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566670541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[0]R generation[0..12]: [87, 108, 110, 111, 108, 108, 108, 109, 109, 110, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566685021Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566699681Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[1]R flush_numbers[0..12]: [176, 221, 224, 225, 221, 221, 220, 222, 222, 223, 225, 225]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566713812Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[1]R generation[0..12]: [87, 108, 110, 111, 108, 108, 108, 109, 109, 110, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566727672Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566741822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[2]R flush_numbers[0..12]: [176, 221, 224, 225, 221, 221, 220, 222, 222, 223, 225, 225]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566755742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[2]R generation[0..12]: [87, 108, 110, 111, 108, 108, 108, 109, 109, 110, 111, 111]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566769552Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566788253Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Max found gen is 112","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2024-07-10T15:49:04.566806053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566819573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566832693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566845763Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"":"downstairs","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566859074Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566871244Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"0","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566884534Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"1","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566897704Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"client":"2","":"downstairs","session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566911084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566923464Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"0eb2f793-7a93-4c7c-a6d1-65a0f3bfd0c7 is now active with session: 2bd287d9-c922-46eb-baee-87622c5d5f1e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566936335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566948585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630,"session_id":"2bd287d9-c922-46eb-baee-87622c5d5f1e"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:04.566960735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 314 len 12288 data: 6 5 6 02/30 Write block 61 len 24576 data: 10 7 7 9 12 10 03/30 Flush 04/30 Write block 562 len 4096 data: 7 05/30 Write block 302 len 16384 data: 6 8 7 7 06/30 Write block 551 len 28672 data: 12 11 9 9 8 5 9 07/30 Read block 116 len 20480 08/30 Write block 275 len 4096 data: 6 09/30 Read block 130 len 8192 10/30 Flush 11/30 Flush 12/30 Write block 564 len 36864 data: 7 6 7 8 8 4 7 6 6 13/30 Write block 221 len 40960 data: 11 12 10 12 11 12 13 12 11 8 14/30 Write block 70 len 40960 data: 11 10 9 11 9 9 8 9 9 8 15/30 Write block 272 len 8192 data: 5 5 16/30 Read block 176 len 28672 17/30 Flush 18/30 Write block 424 len 32768 data: 9 9 8 9 9 10 9 9 19/30 Read block 404 len 36864 20/30 Read block 129 len 28672 21/30 Read block 383 len 8192 22/30 Write block 232 len 24576 data: 6 8 13 14 14 12 23/30 Read block 377 len 40960 24/30 Read block 115 len 24576 25/30 Write block 462 len 36864 data: 6 7 7 7 6 5 6 4 4 26/30 Read block 449 len 16384 27/30 Write block 416 len 28672 data: 3 3 7 5 5 5 7 28/30 Write block 4 len 20480 data: 9 9 8 7 8 29/30 Read block 328 len 24576 30/30 Write block 190 len 12288 data: 6 8 10 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Read 9 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:49:04.593951417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 176 176 176 T T T 1 020-039 108 108 108 221 221 221 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 112 112 112 229 229 229 F F F 4 080-099 108 108 108 221 221 221 F F F 5 100-119 108 108 108 221 221 221 F F F 6 120-139 108 108 108 220 220 220 F F F 7 140-159 109 109 109 222 222 222 F F F 8 160-179 109 109 109 222 222 222 F F F 9 180-199 110 110 110 223 223 223 T T T 10 200-219 111 111 111 225 225 225 F F F 11 220-239 112 112 112 229 229 229 T T T 12 240-259 108 108 108 221 221 221 F F F 13 260-279 112 112 112 229 229 229 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 112 112 112 227 227 227 F F F 16 320-339 108 108 108 221 221 221 F F F 17 340-359 106 106 106 217 217 217 F F F 18 360-379 110 110 110 224 224 224 F F F 19 380-399 110 110 110 223 223 223 F F F 20 400-419 109 109 109 222 222 222 T T T 21 420-439 102 102 102 208 208 208 T T T 22 440-459 109 109 109 222 222 222 F F F 23 460-479 103 103 103 211 211 211 T T T 24 480-499 109 109 109 222 222 222 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 112 112 112 227 227 227 F F F 28 560-579 112 112 112 229 229 229 F F F 29 580-599 110 110 110 223 223 223 F F F Max gen: 112, Max flush: 229 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 230 230 230 F F F 1 020-039 108 108 108 221 221 221 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 112 112 112 229 229 229 F F F 4 080-099 108 108 108 221 221 221 F F F 5 100-119 108 108 108 221 221 221 F F F 6 120-139 108 108 108 220 220 220 F F F 7 140-159 109 109 109 222 222 222 F F F 8 160-179 109 109 109 222 222 222 F F F 9 180-199 113 113 113 230 230 230 F F F 10 200-219 111 111 111 225 225 225 F F F 11 220-239 113 113 113 230 230 230 F F F 12 240-259 108 108 108 221 221 221 F F F 13 260-279 112 112 112 229 229 229 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 112 112 112 227 227 227 F F F 16 320-339 108 108 108 221 221 221 F F F 17 340-359 106 106 106 217 217 217 F F F 18 360-379 110 110 110 224 224 224 F F F 19 380-399 110 110 110 223 223 223 F F F 20 400-419 113 113 113 230 230 230 F F F 21 420-439 113 113 113 230 230 230 F F F 22 440-459 109 109 109 222 222 222 F F F 23 460-479 113 113 113 230 230 230 F F F 24 480-499 109 109 109 222 222 222 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 99 99 99 202 202 202 F F F 27 540-559 112 112 112 227 227 227 F F F 28 560-579 112 112 112 229 229 229 F F F 29 580-599 110 110 110 223 223 223 F F F Max gen: 113, Max flush: 230 Begin loop 57 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.834185115Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.834623831Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.834653252Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638} {"msg":"Crucible 720e70c1-6063-4da6-99c0-2c195eb9c6b2 has session id: e0189e8c-75a8-47f2-b15e-a1005efb82c0","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835076707Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Upstairs opts: Upstairs UUID: 720e70c1-6063-4da6-99c0-2c195eb9c6b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835124618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Crucible stats registered with UUID: 720e70c1-6063-4da6-99c0-2c195eb9c6b2","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835152518Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835461572Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638} {"msg":"720e70c1-6063-4da6-99c0-2c195eb9c6b2 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835492802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835516882Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835540443Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835562223Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835585313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835608884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835634604Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835661234Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835688054Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.835714535Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:49:18.835893467Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:49:18.836032539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:53571)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.836054699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:49:18.83608418Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.83610894Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.8361334Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42032)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.83615854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.836181941Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.836674047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.836706867Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.836732968Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.836759438Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.837370766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.837481877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.837883593Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.837919613Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.837935533Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.838000284Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.838019874Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:18.838037614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.836729488Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:57203)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.83689402Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.838291498Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.838338948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.838910155Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839439512Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839462973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[0]R flush_numbers[0..12]: [230, 221, 224, 229, 221, 221, 220, 222, 222, 230, 225, 230]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839478273Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[0]R generation[0..12]: [113, 108, 110, 112, 108, 108, 108, 109, 109, 113, 111, 113]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839492803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839507403Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[1]R flush_numbers[0..12]: [230, 221, 224, 229, 221, 221, 220, 222, 222, 230, 225, 230]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839521293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[1]R generation[0..12]: [113, 108, 110, 112, 108, 108, 108, 109, 109, 113, 111, 113]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839535283Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839549703Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[2]R flush_numbers[0..12]: [230, 221, 224, 229, 221, 221, 220, 222, 222, 230, 225, 230]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839568854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[2]R generation[0..12]: [113, 108, 110, 112, 108, 108, 108, 109, 109, 113, 111, 113]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839583104Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839597384Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Max found gen is 114","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2024-07-10T15:49:28.839614094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839627824Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839640755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839653675Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"":"downstairs","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839666825Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839679045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"0","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839692385Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"1","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839705965Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"client":"2","":"downstairs","session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839719396Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839731736Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"720e70c1-6063-4da6-99c0-2c195eb9c6b2 is now active with session: e0189e8c-75a8-47f2-b15e-a1005efb82c0","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839744856Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839756826Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638,"session_id":"e0189e8c-75a8-47f2-b15e-a1005efb82c0"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:28.839772136Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 491 len 32768 data: 9 9 9 9 8 9 10 10 02/30 Read block 145 len 28672 03/30 Write block 322 len 36864 data: 11 11 11 10 11 10 9 9 9 04/30 Read block 124 len 24576 05/30 Read block 509 len 32768 06/30 Write block 130 len 36864 data: 13 12 11 9 11 11 10 13 14 07/30 Read block 61 len 32768 08/30 Write block 103 len 36864 data: 7 6 4 4 3 5 5 4 4 09/30 Write block 198 len 12288 data: 11 9 9 10/30 Write block 130 len 40960 data: 14 13 12 10 12 12 11 14 15 15 11/30 Write block 452 len 8192 data: 12 11 12/30 Read block 283 len 36864 13/30 Write block 143 len 20480 data: 12 12 10 11 13 14/30 Write block 528 len 8192 data: 12 12 15/30 Write block 524 len 36864 data: 6 7 13 12 13 13 12 13 11 16/30 Read block 109 len 40960 17/30 Read block 273 len 24576 18/30 Read block 252 len 32768 19/30 Write block 188 len 20480 data: 6 7 7 9 11 20/30 Write block 213 len 24576 data: 8 7 6 7 6 6 21/30 Write block 65 len 20480 data: 13 11 8 9 9 22/30 Write block 98 len 40960 data: 8 8 10 9 7 8 7 5 5 4 23/30 Write block 138 len 8192 data: 16 16 24/30 Read block 262 len 28672 25/30 Read block 99 len 16384 26/30 Read block 327 len 8192 27/30 Write block 339 len 40960 data: 10 10 11 12 12 13 12 12 14 13 28/30 Write block 458 len 16384 data: 3 3 4 5 29/30 Read block 227 len 8192 30/30 Read block 143 len 36864 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false 31 Acked 1030 Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:49:28.866307283Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 230 230 230 F F F 1 020-039 108 108 108 221 221 221 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 114 114 114 231 231 231 F F F 4 080-099 114 114 114 231 231 231 F F F 5 100-119 114 114 114 231 231 231 F F F 6 120-139 114 114 114 231 231 231 F F F 7 140-159 114 114 114 231 231 231 F F F 8 160-179 109 109 109 222 222 222 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 114 114 114 231 231 231 F F F 11 220-239 113 113 113 230 230 230 F F F 12 240-259 108 108 108 221 221 221 F F F 13 260-279 112 112 112 229 229 229 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 112 112 112 227 227 227 F F F 16 320-339 114 114 114 231 231 231 T T T 17 340-359 106 106 106 217 217 217 T T T 18 360-379 110 110 110 224 224 224 F F F 19 380-399 110 110 110 223 223 223 F F F 20 400-419 113 113 113 230 230 230 F F F 21 420-439 113 113 113 230 230 230 F F F 22 440-459 114 114 114 231 231 231 T T T 23 460-479 113 113 113 230 230 230 T T T 24 480-499 114 114 114 231 231 231 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 114 114 114 231 231 231 F F F 27 540-559 112 112 112 227 227 227 F F F 28 560-579 112 112 112 229 229 229 F F F 29 580-599 110 110 110 223 223 223 F F F Max gen: 114, Max flush: 231 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 230 230 230 F F F 1 020-039 108 108 108 221 221 221 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 114 114 114 231 231 231 F F F 4 080-099 114 114 114 231 231 231 F F F 5 100-119 114 114 114 231 231 231 F F F 6 120-139 114 114 114 231 231 231 F F F 7 140-159 114 114 114 231 231 231 F F F 8 160-179 109 109 109 222 222 222 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 114 114 114 231 231 231 F F F 11 220-239 113 113 113 230 230 230 F F F 12 240-259 108 108 108 221 221 221 F F F 13 260-279 112 112 112 229 229 229 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 112 112 112 227 227 227 F F F 16 320-339 115 115 115 232 232 232 F F F 17 340-359 115 115 115 232 232 232 F F F 18 360-379 110 110 110 224 224 224 F F F 19 380-399 110 110 110 223 223 223 F F F 20 400-419 113 113 113 230 230 230 F F F 21 420-439 113 113 113 230 230 230 F F F 22 440-459 115 115 115 232 232 232 F F F 23 460-479 115 115 115 232 232 232 F F F 24 480-499 114 114 114 231 231 231 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 114 114 114 231 231 231 F F F 27 540-559 112 112 112 227 227 227 F F F 28 560-579 112 112 112 229 229 229 F F F 29 580-599 110 110 110 223 223 223 F F F Max gen: 115, Max flush: 232 Begin loop 58 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.079864943Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.080247838Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.080266428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646} {"msg":"Crucible b22c2cc2-98f3-4646-857d-23349327563f has session id: 9e3277d0-5abc-4c53-bd1c-31032a0bfd78","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.080509561Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Upstairs opts: Upstairs UUID: b22c2cc2-98f3-4646-857d-23349327563f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.080552582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Crucible stats registered with UUID: b22c2cc2-98f3-4646-857d-23349327563f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.080578922Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.080866636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646} {"msg":"b22c2cc2-98f3-4646-857d-23349327563f active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081010568Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081028218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081042948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081056738Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081070679Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081084189Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081105429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081130609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.08114636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.08116051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:49:43.081363502Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_connection connected from Ok(127.0.0.1:55994)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081401693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_connection connected from Ok(127.0.0.1:38355)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081434023Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:49:43.081527304Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:49:43.081568155Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081593475Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081617866Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.081721747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.082225223Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.082259083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.082322784Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.082349375Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.082892302Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.082920642Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.083478819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.083506179Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.08352097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.08353761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.08355181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:43.08356522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.082887042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_connection connected from Ok(127.0.0.1:60337)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.083109454Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.084266339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.08430631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.084781116Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085319802Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085339873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[0]R flush_numbers[0..12]: [230, 221, 224, 231, 231, 231, 231, 231, 222, 231, 231, 230]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085355173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[0]R generation[0..12]: [113, 108, 110, 114, 114, 114, 114, 114, 109, 114, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085369983Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085384383Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[1]R flush_numbers[0..12]: [230, 221, 224, 231, 231, 231, 231, 231, 222, 231, 231, 230]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085398614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[1]R generation[0..12]: [113, 108, 110, 114, 114, 114, 114, 114, 109, 114, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085412764Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085432454Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[2]R flush_numbers[0..12]: [230, 221, 224, 231, 231, 231, 231, 231, 222, 231, 231, 230]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085446934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[2]R generation[0..12]: [113, 108, 110, 114, 114, 114, 114, 114, 109, 114, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085460984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085475184Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085488225Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS",Wait for a query_work_queue command to finish before sending IO "pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085505375Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085518535Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085531685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"":"downstairs","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085545175Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085557316Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"0","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085570906Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"1","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085584316Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"client":"2","":"downstairs","session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085598016Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085610456Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"b22c2cc2-98f3-4646-857d-23349327563f is now active with session: 9e3277d0-5abc-4c53-bd1c-31032a0bfd78","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085623366Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085638046Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646,"session_id":"9e3277d0-5abc-4c53-bd1c-31032a0bfd78"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:49:53.085650607Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 91 len 12288 data: 8 8 8 02/30 Write block 396 len 40960 data: 11 12 12 13 14 11 10 10 8 8 03/30 Read block 39 len 32768 04/30 Write block 486 len 16384 data: 7 7 7 7 05/30 Write block 119 len 36864 data: 8 9 7 7 6 7 7 6 6 06/30 Read block 101 len 4096 07/30 Read block 297 len 12288 08/30 Read block 48 len 4096 09/30 Flush 10/30 Write block 268 len 8192 data: 2 3 11/30 Write block 18 len 20480 data: 9 9 9 9 8 12/30 Flush 13/30 Write block 552 len 8192 data: 12 10 14/30 Flush 15/30 Write block 103 len 4096 data: 9 16/30 Write block 374 len 36864 data: 13 10 12 15 14 12 11 9 9 17/30 Read block 262 len 8192 18/30 Read block 384 len 24576 19/30 Read block 438 len 40960 20/30 Read block 56 len 36864 21/30 Read block 360 len 20480 22/30 Read block 110 len 40960 23/30 Read block 93 len 4096 24/30 Read block 7 len 12288 25/30 Read block 406 len 36864 26/30 Write block 236 len 20480 data: 15 13 10 10 8 27/30 Read block 331 len 36864 28/30 Read block 302 len 8192 29/30 Read block 372 len 40960 30/30 Write block 582 len 28672 data: 7 7 5 7 9 8 9 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Read 6 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Read 9 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:49:53.114000557Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 234 234 234 F F F 1 020-039 116 116 116 234 234 234 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 114 114 114 231 231 231 F F F 4 080-099 116 116 116 233 233 233 F F F 5 100-119 116 116 116 236 236 236 F F F 6 120-139 116 116 116 233 233 233 F F F 7 140-159 114 114 114 231 231 231 F F F 8 160-179 109 109 109 222 222 222 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 114 114 114 231 231 231 F F F 11 220-239 113 113 113 230 230 230 T T T 12 240-259 108 108 108 221 221 221 T T T 13 260-279 116 116 116 234 234 234 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 112 112 112 227 227 227 F F F 16 320-339 115 115 115 232 232 232 F F F 17 340-359 115 115 115 232 232 232 F F F 18 360-379 116 116 116 236 236 236 F F F 19 380-399 116 116 116 236 236 236 F F F 20 400-419 116 116 116 233 233 233 F F F 21 420-439 113 113 113 230 230 230 F F F 22 440-459 115 115 115 232 232 232 F F F 23 460-479 115 115 115 232 232 232 F F F 24 480-499 116 116 116 233 233 233 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 114 114 114 231 231 231 F F F 27 540-559 116 116 116 235 235 235 F F F 28 560-579 112 112 112 229 229 229 F F F 29 580-599 110 110 110 223 223 223 T T T Max gen: 116, Max flush: 236 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 234 234 234 F F F 1 020-039 116 116 116 234 234 234 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 114 114 114 231 231 231 F F F 4 080-099 116 116 116 233 233 233 F F F 5 100-119 116 116 116 236 236 236 F F F 6 120-139 116 116 116 233 233 233 F F F 7 140-159 114 114 114 231 231 231 F F F 8 160-179 109 109 109 222 222 222 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 114 114 114 231 231 231 F F F 11 220-239 117 117 117 237 237 237 F F F 12 240-259 117 117 117 237 237 237 F F F 13 260-279 116 116 116 234 234 234 F F F 14 280-299 104 104 104 213 213 213 F F F 15 300-319 112 112 112 227 227 227 F F F 16 320-339 115 115 115 232 232 232 F F F 17 340-359 115 115 115 232 232 232 F F F 18 360-379 116 116 116 236 236 236 F F F 19 380-399 116 116 116 236 236 236 F F F 20 400-419 116 116 116 233 233 233 F F F 21 420-439 113 113 113 230 230 230 F F F 22 440-459 115 115 115 232 232 232 F F F 23 460-479 115 115 115 232 232 232 F F F 24 480-499 116 116 116 233 233 233 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 114 114 114 231 231 231 F F F 27 540-559 116 116 116 235 235 235 F F F 28 560-579 112 112 112 229 229 229 F F F 29 580-599 117 117 117 237 237 237 F F F Max gen: 117, Max flush: 237 Begin loop 59 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.319333422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.319883909Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.31991633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654} {"msg":"Crucible 6fcdde95-a79f-4697-84d3-3af4d8c11732 has session id: 6280fc9e-dad6-49da-abdb-84f247fa37bd","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320182393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Upstairs opts: Upstairs UUID: 6fcdde95-a79f-4697-84d3-3af4d8c11732, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320230503Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Crucible stats registered with UUID: 6fcdde95-a79f-4697-84d3-3af4d8c11732","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320254804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320637809Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654} {"msg":"6fcdde95-a79f-4697-84d3-3af4d8c11732 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320674489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320689859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.32070452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.32071871Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.32074166Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.32076672Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320792401Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320817961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320842701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320865391Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:61849)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.320973723Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:61533)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.321092645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:07.321219946Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:07.321248587Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:07.321266257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.321279217Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.321292637Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.321441959Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.322045806Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.322068007Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.322082067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.322095577Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.322777626Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.322811476Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.323380354Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.323409754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.323435654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.323460455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.323478115Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:07.323494845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.32236869Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58941)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.322668324Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.323673097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.323704878Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324177584Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.32469266Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.32471524Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[0]R flush_numbers[0..12]: [234, 234, 224, 231, 233, 236, 233, 231, 222, 231, 231, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.3247299Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[0]R generation[0..12]: [116, 116, 110, 114, 116, 116, 116, 114, 109, 114, 114, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324744431Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324759261Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[1]R flush_numbers[0..12]: [234, 234, 224, 231, 233, 236, 233, 231, 222, 231, 231, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324773811Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[1]R generation[0..12]: [116, 116, 110, 114, 116, 116, 116, 114, 109, 114, 114, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324793051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324808052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[2]R flush_numbers[0..12]: [234, 234, 224, 231, 233, 236, 233, 231, 222, 231, 231, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324822092Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[2]R generation[0..12]: [116, 116, 110, 114, 116, 116, 116, 114, 109, 114, 114, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324836092Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324850242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324863322Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324885402Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324901863Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324918183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"":"downstairs","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324934993Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324950123Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"0","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324966923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"1","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.324983384Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"client":"2","":"downstairs","session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.325000104Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.325018154Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"6fcdde95-a79f-4697-84d3-3af4d8c11732 is now active with session: 6280fc9e-dad6-49da-abdb-84f247fa37bd","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.325034914Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.325049975Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654,"session_id":"6280fc9e-dad6-49da-abdb-84f247fa37bd"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:17.325065355Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 470 len 32768 data: 5 4 4 7 7 11 15 14 02/30 Flush 03/30 Write block 367 len 16384 data: 5 9 11 14 04/30 Write block 487 len 8192 data: 8 8 05/30 Write block 293 len 40960 data: 8 8 5 6 6 6 5 5 5 7 06/30 Read block 598 len 4096 07/30 Write block 520 len 40960 data: 5 5 5 6 7 8 14 13 14 14 08/30 Write block 487 len 24576 data: 9 9 8 8 10 10 09/30 Write block 426 len 16384 data: 9 10 10 11 10/30 Write block 318 len 8192 data: 7 9 11/30 Read block 594 len 12288 12/30 Flush 13/30 Write block 202 len 12288 data: 6 7 7 14/30 Write block 11 len 40960 data: 6 5 6 6 5 7 7 10 10 10 15/30 Read block 170 len 16384 16/30 Write block 589 len 12288 data: 8 11 12 17/30 Read block 91 len 8192 18/30 Write block 166 len 16384 data: 8 9 8 6 19/30 Read block 488 len 16384 20/30 Read block 95 len 20480 21/30 Write block 379 len 8192 data: 13 12 22/30 Flush 23/30 Flush 24/30 Write block 437 len 12288 data: 8 8 10 25/30 Read block 197 len 12288 26/30 Write block 143 len 20480 data: 13 13 11 12 14 27/30 Read block 139 len 20480 28/30 Read block 366 len 40960 29/30 Write block 4 len 32768 data: 10 10 9 8 9 7 6 7 30/30 Write block 527 len 24576 data: 14 15 15 13 14 12 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 8 Done Done Done false 31 Acked 1030 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:17.353441645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 240 240 240 T T T 1 020-039 118 118 118 240 240 240 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 114 114 114 231 231 231 F F F 4 080-099 116 116 116 233 233 233 F F F 5 100-119 116 116 116 236 236 236 F F F 6 120-139 116 116 116 233 233 233 F F F 7 140-159 114 114 114 231 231 231 T T T 8 160-179 118 118 118 241 241 241 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 117 117 117 237 237 237 F F F 12 240-259 117 117 117 237 237 237 F F F 13 260-279 116 116 116 234 234 234 F F F 14 280-299 118 118 118 239 239 239 F F F 15 300-319 118 118 118 239 239 239 F F F 16 320-339 115 115 115 232 232 232 F F F 17 340-359 115 115 115 232 232 232 F F F 18 360-379 118 118 118 241 241 241 F F F 19 380-399 118 118 118 241 241 241 F F F 20 400-419 116 116 116 233 233 233 F F F 21 420-439 118 118 118 239 239 239 T T T 22 440-459 115 115 115 232 232 232 F F F 23 460-479 118 118 118 238 238 238 F F F 24 480-499 118 118 118 239 239 239 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 118 118 118 239 239 239 T T T 27 540-559 116 116 116 235 235 235 F F F 28 560-579 112 112 112 229 229 229 F F F 29 580-599 118 118 118 241 241 241 F F F Max gen: 118, Max flush: 241 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 242 242 242 F F F 1 020-039 118 118 118 240 240 240 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 114 114 114 231 231 231 F F F 4 080-099 116 116 116 233 233 233 F F F 5 100-119 116 116 116 236 236 236 F F F 6 120-139 116 116 116 233 233 233 F F F 7 140-159 119 119 119 242 242 242 F F F 8 160-179 118 118 118 241 241 241 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 117 117 117 237 237 237 F F F 12 240-259 117 117 117 237 237 237 F F F 13 260-279 116 116 116 234 234 234 F F F 14 280-299 118 118 118 239 239 239 F F F 15 300-319 118 118 118 239 239 239 F F F 16 320-339 115 115 115 232 232 232 F F F 17 340-359 115 115 115 232 232 232 F F F 18 360-379 118 118 118 241 241 241 F F F 19 380-399 118 118 118 241 241 241 F F F 20 400-419 116 116 116 233 233 233 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 115 115 115 232 232 232 F F F 23 460-479 118 118 118 238 238 238 F F F 24 480-499 118 118 118 239 239 239 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 119 119 119 242 242 242 F F F 27 540-559 116 116 116 235 235 235 F F F 28 560-579 112 112 112 229 229 229 F F F 29 580-599 118 118 118 241 241 241 F F F Max gen: 119, Max flush: 242 Begin loop 60 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.567742035Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.56818351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.568216541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662} {"msg":"Crucible 702b7f21-5ec9-45b2-b6b4-6e08a2989991 has session id: ebac0a69-9e02-4235-aed8-25b2740cc879","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.568291731Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Upstairs opts: Upstairs UUID: 702b7f21-5ec9-45b2-b6b4-6e08a2989991, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.568331302Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Crucible stats registered with UUID: 702b7f21-5ec9-45b2-b6b4-6e08a2989991","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.568356232Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.568857569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662} {"msg":"702b7f21-5ec9-45b2-b6b4-6e08a2989991 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.568919289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.56894931Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.56897677Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.56900238Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569026241Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569041521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569055511Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569077611Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569092212Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569106552Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:31.569267954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:31.569309524Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_connection connected from Ok(127.0.0.1:42881)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569334895Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:31.569358535Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569378105Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569400716Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_connection connected from Ok(127.0.0.1:62026)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569558807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.569595128Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.570152645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.570190286Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.570214556Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.570236956Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.570709352Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.570826743Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.571187338Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.571229769Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.571258389Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.571284279Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.57131133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:31.57133374Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.571104477Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_connection connected from Ok(127.0.0.1:48396)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.571459782Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.572227981Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.572255242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.572775788Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573291605Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573319835Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[0]R flush_numbers[0..12]: [242, 240, 224, 231, 233, 236, 233, 242, 241, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573339926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[0]R generation[0..12]: [119, 118, 110, 114, 116, 116, 116, 119, 118, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573358266Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573376186Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[1]R flush_numbers[0..12]: [242, 240, 224, 231, 233, 236, 233, 242, 241, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573399556Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[1]R generation[0..12]: [119, 118, 110, 114, 116, 116, 116, 119, 118, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573417367Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573435107Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[2]R flush_numbers[0..12]: [242, 240, 224, 231, 233, 236, 233, 242, 241, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573452387Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[2]R generation[0..12]: [119, 118, 110, 114, 116, 116, 116, 119, 118, 114, 118, 117]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:50:41.573469817Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573487897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573504488Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573521148Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573541568Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573570069Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"":"downstairs","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573600469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573617489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"0","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573644289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"1","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.57367427Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"client":"2","":"downstairs","session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.5737203Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573749521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"702b7f21-5ec9-45b2-b6b4-6e08a2989991 is now active with session: ebac0a69-9e02-4235-aed8-25b2740cc879","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-07-10T15:50:41.573777761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573794581Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662,"session_id":"ebac0a69-9e02-4235-aed8-25b2740cc879"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:41.573810201Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 264 len 16384 data: 7 7 7 5 03/30 Read block 129 len 40960 04/30 Read block 317 len 4096 05/30 Read block 213 len 28672 06/30 Read block 210 len 8192 07/30 Write block 479 len 4096 data: 12 08/30 Write block 93 len 32768 data: 9 7 8 8 8 9 9 11 09/30 Write block 169 len 32768 data: 7 6 5 6 8 9 7 5 10/30 Write block 73 len 32768 data: 12 10 10 9 10 10 9 7 11/30 Flush 12/30 Write block 538 len 28672 data: 12 9 9 6 7 8 10 13/30 Read block 230 len 8192 14/30 Write block 309 len 8192 data: 7 7 15/30 Read block 532 len 36864 16/30 Write block 5 len 28672 data: 11 10 9 10 8 7 8 17/30 Read block 253 len 24576 18/30 Write block 561 len 28672 data: 5 8 6 8 7 8 9 19/30 Write block 571 len 40960 data: 7 7 6 8 8 11 10 12 10 9 20/30 Write block 328 len 4096 data: 10 21/30 Write block 356 len 4096 data: 11 22/30 Read block 528 len 40960 23/30 Write block 408 len 24576 data: 8 7 6 4 4 6 24/30 Read block 119 len 40960 25/30 Read block 502 len 8192 26/30 Read block 404 len 28672 27/30 Write block 13 len 36864 data: 7 7 6 8 8 11 11 11 10 28/30 Write block 588 len 36864 data: 10 9 12 13 10 7 8 7 6 29/30 Write block 589 len 16384 data: 10 13 14 11 30/30 Write block 292 len 8192 data: 8 9 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 10 Done Done Done false 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Write 9 Done Done Done false 30 Acked 1029 Write 4 Done Done Sent false 31 Acked 1030 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 2 4 Done 11 11 10 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:41.59887007Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 245 245 245 T T T 1 020-039 118 118 118 240 240 240 T T T 2 040-059 110 110 110 224 224 224 F F F 3 060-079 120 120 120 244 244 244 F F F 4 080-099 120 120 120 244 244 244 F F F 5 100-119 120 120 120 244 244 244 F F F 6 120-139 116 116 116 233 233 233 F F F 7 140-159 119 119 119 242 242 242 F F F 8 160-179 120 120 120 244 244 244 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 117 117 117 237 237 237 F F F 12 240-259 117 117 117 237 237 237 F F F 13 260-279 120 120 120 244 244 244 F F F 14 280-299 118 118 118 239 239 239 T T T 15 300-319 120 120 120 245 245 245 F F F 16 320-339 115 115 115 232 232 232 T T T 17 340-359 115 115 115 232 232 232 T T T 18 360-379 118 118 118 241 241 241 F F F 19 380-399 118 118 118 241 241 241 F F F 20 400-419 116 116 116 233 233 233 T T T 21 420-439 119 119 119 242 242 242 F F F 22 440-459 115 115 115 232 232 232 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 118 118 118 239 239 239 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 120 120 120 245 245 245 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 120 120 120 245 245 245 T T T 29 580-599 118 118 118 241 241 241 T T T Max gen: 120, Max flush: 245 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 246 246 246 F F F 1 020-039 121 121 121 246 246 246 F F F 2 040-059 110 110 110 224 224 224 F F F 3 060-079 120 120 120 244 244 244 F F F 4 080-099 120 120 120 244 244 244 F F F 5 100-119 120 120 120 244 244 244 F F F 6 120-139 116 116 116 233 233 233 F F F 7 140-159 119 119 119 242 242 242 F F F 8 160-179 120 120 120 244 244 244 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 117 117 117 237 237 237 F F F 12 240-259 117 117 117 237 237 237 F F F 13 260-279 120 120 120 244 244 244 F F F 14 280-299 121 121 121 246 246 246 F F F 15 300-319 120 120 120 245 245 245 F F F 16 320-339 121 121 121 246 246 246 F F F 17 340-359 121 121 121 246 246 246 F F F 18 360-379 118 118 118 241 241 241 F F F 19 380-399 118 118 118 241 241 241 F F F 20 400-419 121 121 121 246 246 246 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 115 115 115 232 232 232 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 118 118 118 239 239 239 F F F 25 500-519 108 108 108 221 221 221 F F F 26 520-539 120 120 120 245 245 245 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 121 121 121 246 246 246 F F F 29 580-599 121 121 121 246 246 246 F F F Max gen: 121, Max flush: 246 Begin loop 61 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.865065007Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.865447072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.865466312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670} {"msg":"Crucible 46d5b786-06cb-4137-90e7-0f8d8634e734 has session id: 07755ef1-cc2b-4e79-b704-33870584f312","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.865811507Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Upstairs opts: Upstairs UUID: 46d5b786-06cb-4137-90e7-0f8d8634e734, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.865858067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Crucible stats registered with UUID: 46d5b786-06cb-4137-90e7-0f8d8634e734","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.865878347Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866286233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670} {"msg":"46d5b786-06cb-4137-90e7-0f8d8634e734 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866308543Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866321923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866336143Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866349844Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866363434Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866376814Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866390554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866420474Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866444255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866459585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_connection connected from Ok(127.0.0.1:46124)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866655907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_connection connected from Ok(127.0.0.1:41225)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.866694708Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:55.866801119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:55.86683881Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:50:55.866865Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.86687881Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.86689243Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.867028102Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.867549909Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.867576899Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.867592039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.867605889Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.868195697Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.868315609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.868684443Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.868722354Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.868752344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.868780414Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.868808015Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:50:55.868830255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.867846873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_connection connected from Ok(127.0.0.1:42033)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.868073675Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.869065678Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.869095548Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.869679766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870240173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870273814Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[0]R flush_numbers[0..12]: [246, 246, 224, 244, 244, 244, 233, 242, 244, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870294444Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[0]R generation[0..12]: [121, 121, 110, 120, 120, 120, 116, 119, 120, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870318844Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870337894Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[1]R flush_numbers[0..12]: [246, 246, 224, 244, 244, 244, 233, 242, 244, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870355724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[1]R generation[0..12]: [121, 121, 110, 120, 120, 120, 116, 119, 120, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870373395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870391195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[2]R flush_numbers[0..12]: [246, 246, 224, 244, 244, 244, 233, 242, 244, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870408815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,Wait for a query_work_queue command to finish before sending IO "":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[2]R generation[0..12]: [121, 121, 110, 120, 120, 120, 116, 119, 120, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870430916Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870448876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870465306Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870482086Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870498346Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870514866Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"":"downstairs","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870531657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870546777Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"0","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870566407Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"1","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870583267Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"client":"2","":"downstairs","session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870600018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870615438Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"46d5b786-06cb-4137-90e7-0f8d8634e734 is now active with session: 07755ef1-cc2b-4e79-b704-33870584f312","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870631588Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-07-10T15:51:05.870646708Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670,"session_id":"07755ef1-cc2b-4e79-b704-33870584f312"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:05.870662618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 446 len 24576 data: 9 12 11 10 9 11 02/30 Write block 59 len 12288 data: 9 9 11 03/30 Read block 541 len 4096 04/30 Write block 22 len 32768 data: 9 6 6 7 7 7 9 10 05/30 Write block 127 len 32768 data: 7 6 8 15 14 13 11 13 06/30 Read block 329 len 36864 07/30 Read block 164 len 12288 08/30 Write block 103 len 36864 data: 10 8 6 6 5 6 6 5 5 09/30 Read block 270 len 20480 10/30 Write block 498 len 16384 data: 11 11 12 11 11/30 Read block 422 len 24576 12/30 Write block 451 len 32768 data: 12 13 12 8 8 7 4 4 13/30 Write block 241 len 24576 data: 10 9 10 12 10 11 14/30 Write block 313 len 16384 data: 5 7 6 7 15/30 Read block 110 len 36864 16/30 Read block 565 len 12288 17/30 Flush 18/30 Write block 103 len 36864 data: 11 9 7 7 6 7 7 6 6 19/30 Read block 484 len 16384 20/30 Read block 90 len 8192 21/30 Read block 241 len 40960 22/30 Write block 81 len 28672 data: 6 6 7 7 8 6 9 23/30 Read block 227 len 8192 24/30 Read block 5 len 8192 25/30 Write block 318 len 40960 data: 8 10 9 9 12 12 12 11 12 11 26/30 Write block 289 len 36864 data: 8 6 7 9 10 9 6 7 7 27/30 Write block 560 len 36864 data: 5 6 9 7 9 8 9 10 9 28/30 Read block 315 len 24576 29/30 Read block 181 len 8192 30/30 Read block 521 len 24576 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false 31 Acked 1030 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:51:05.898113777Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 246 246 246 F F F 1 020-039 122 122 122 247 247 247 F F F 2 040-059 122 122 122 247 247 247 F F F 3 060-079 122 122 122 247 247 247 F F F 4 080-099 120 120 120 244 244 244 T T T 5 100-119 122 122 122 248 248 248 F F F 6 120-139 122 122 122 247 247 247 F F F 7 140-159 119 119 119 242 242 242 F F F 8 160-179 120 120 120 244 244 244 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 117 117 117 237 237 237 F F F 12 240-259 122 122 122 247 247 247 F F F 13 260-279 120 120 120 244 244 244 F F F 14 280-299 121 121 121 246 246 246 T T T 15 300-319 122 122 122 247 247 247 T T T 16 320-339 121 121 121 246 246 246 T T T 17 340-359 121 121 121 246 246 246 F F F 18 360-379 118 118 118 241 241 241 F F F 19 380-399 118 118 118 241 241 241 F F F 20 400-419 121 121 121 246 246 246 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 122 122 122 247 247 247 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 122 122 122 247 247 247 F F F 25 500-519 122 122 122 247 247 247 F F F 26 520-539 120 120 120 245 245 245 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 121 121 121 246 246 246 T T T 29 580-599 121 121 121 246 246 246 F F F Max gen: 122, Max flush: 248 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 246 246 246 F F F 1 020-039 122 122 122 247 247 247 F F F 2 040-059 122 122 122 247 247 247 F F F 3 060-079 122 122 122 247 247 247 F F F 4 080-099 123 123 123 249 249 249 F F F 5 100-119 122 122 122 248 248 248 F F F 6 120-139 122 122 122 247 247 247 F F F 7 140-159 119 119 119 242 242 242 F F F 8 160-179 120 120 120 244 244 244 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 117 117 117 237 237 237 F F F 12 240-259 122 122 122 247 247 247 F F F 13 260-279 120 120 120 244 244 244 F F F 14 280-299 123 123 123 249 249 249 F F F 15 300-319 123 123 123 249 249 249 F F F 16 320-339 123 123 123 249 249 249 F F F 17 340-359 121 121 121 246 246 246 F F F 18 360-379 118 118 118 241 241 241 F F F 19 380-399 118 118 118 241 241 241 F F F 20 400-419 121 121 121 246 246 246 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 122 122 122 247 247 247 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 122 122 122 247 247 247 F F F 25 500-519 122 122 122 247 247 247 F F F 26 520-539 120 120 120 245 245 245 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 121 121 121 246 246 246 F F F Max gen: 123, Max flush: 249 Begin loop 62 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.129965109Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.130320163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.130348744Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678} {"msg":"Crucible 7295e6b7-dd3a-4a32-acb3-0ddc58dd70b9 has session id: 4456b11c-3430-4d1b-b7aa-89e5386ab47f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.130721499Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Upstairs opts: Upstairs UUID: 7295e6b7-dd3a-4a32-acb3-0ddc58dd70b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.130755369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Crucible stats registered with UUID: 7295e6b7-dd3a-4a32-acb3-0ddc58dd70b9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.130770889Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131227255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678} {"msg":"7295e6b7-dd3a-4a32-acb3-0ddc58dd70b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131257465Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131271566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131286106Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131300176Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131314166Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131327726Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131345766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131439577Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131458908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131472968Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:51:20.131761412Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:51:20.131803452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:51:20.131836373Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131858313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131880803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40081)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131896153Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131910234Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35839)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.131924994Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.132504751Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.132531791Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.132549232Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.132571042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.133198Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.13323445Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.133737837Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.133764157Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.133778317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.133790758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.133804347Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:20.133817748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.132566772Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44817)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.132690293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.133810798Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.133842618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.134504067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135032273Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135064854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[0]R flush_numbers[0..12]: [246, 247, 247, 247, 249, 248, 247, 242, 244, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135083554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[0]R generation[0..12]: [121, 122, 122, 122, 123, 122, 122, 119, 120, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135101754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135119844Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[1]R flush_numbers[0..12]: [246, 247, 247, 247, 249, 248, 247, 242, 244, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135137415Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[1]R generation[0..12]: [121, 122, 122, 122, 123, 122, 122, 119, 120, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135154905Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135172665Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[2]R flush_numbers[0..12]: [246, 247, 247, 247, 249, 248, 247, 242, 244, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135190115Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [121, 122, 122, 122, 123, 122, 122, 119, 120, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135210776Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135228486Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135244856Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135261406Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135277736Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135294057Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"":"downstairs","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135313097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135328517Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"0","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135345137Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"1","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135361697Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"client":"2","":"downstairs","session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135378348Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135393668Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"7295e6b7-dd3a-4a32-acb3-0ddc58dd70b9 is now active with session: 4456b11c-3430-4d1b-b7aa-89e5386ab47f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135409568Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:30.135424318Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678,"session_id":"4456b11c-3430-4d1b-b7aa-89e5386ab47f"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-07-10T15:51:30.135439618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 20480 data: 7 7 8 8 8 02/30 Read block 221 len 8192 03/30 Write block 371 len 36864 data: 12 10 12 14 11 13 16 15 14 04/30 Flush 05/30 Write block 377 len 28672 data: 17 16 15 13 10 10 5 06/30 Flush 07/30 Write block 501 len 20480 data: 12 11 12 11 9 08/30 Read block 114 len 4096 09/30 Flush 10/30 Write block 490 len 12288 data: 9 11 11 11/30 Read block 535 len 12288 12/30 Read block 590 len 28672 13/30 Write block 154 len 32768 data: 8 8 10 9 12 11 8 7 14/30 Write block 304 len 20480 data: 8 8 7 8 8 15/30 Flush 16/30 Write block 54 len 28672 data: 8 8 11 9 9 10 10 17/30 Write block 443 len 20480 data: 11 11 9 10 13 18/30 Read block 89 len 40960 19/30 Read block 590 len 8192 20/30 Read block 174 len 32768 21/30 Write block 69 len 20480 data: 10 12 11 10 13 22/30 Read block 431 len 28672 23/30 Write block 389 len 24576 data: 8 9 9 9 11 10 24/30 Read block 272 len 36864 25/30 Read block 290 len 8192 26/30 Read block 531 len 12288 27/30 Read block 371 len 16384 28/30 Write block 395 len 40960 data: 10 12 13 13 14 15 12 11 11 9 29/30 Read block 212 len 16384 30/30 Read block 203 len 12288 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 7 Done Done Done false 17 Acked 1016 Write 5 Done Done Done false 18 Acked 1017 Read 10 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Read 8 Done Done Done false 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Write 6 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:51:30.164381276Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 246 246 246 F F F 1 020-039 124 124 124 250 250 250 F F F 2 040-059 122 122 122 247 247 247 T T T 3 060-079 122 122 122 247 247 247 T T T 4 080-099 123 123 123 249 249 249 F F F 5 100-119 122 122 122 248 248 248 F F F 6 120-139 122 122 122 247 247 247 F F F 7 140-159 124 124 124 253 253 253 F F F 8 160-179 124 124 124 253 253 253 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 117 117 117 237 237 237 F F F 12 240-259 122 122 122 247 247 247 F F F 13 260-279 120 120 120 244 244 244 F F F 14 280-299 123 123 123 249 249 249 F F F 15 300-319 124 124 124 253 253 253 F F F 16 320-339 123 123 123 249 249 249 F F F 17 340-359 121 121 121 246 246 246 F F F 18 360-379 124 124 124 251 251 251 F F F 19 380-399 124 124 124 251 251 251 T T T 20 400-419 121 121 121 246 246 246 T T T 21 420-439 119 119 119 242 242 242 F F F 22 440-459 122 122 122 247 247 247 T T T 23 460-479 120 120 120 244 244 244 F F F 24 480-499 124 124 124 253 253 253 F F F 25 500-519 124 124 124 252 252 252 F F F 26 520-539 120 120 120 245 245 245 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 121 121 121 246 246 246 F F F Max gen: 124, Max flush: 253 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 246 246 246 F F F 1 020-039 124 124 124 250 250 250 F F F 2 040-059 125 125 125 254 254 254 F F F 3 060-079 125 125 125 254 254 254 F F F 4 080-099 123 123 123 249 249 249 F F F 5 100-119 122 122 122 248 248 248 F F F 6 120-139 122 122 122 247 247 247 F F F 7 140-159 124 124 124 253 253 253 F F F 8 160-179 124 124 124 253 253 253 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 117 117 117 237 237 237 F F F 12 240-259 122 122 122 247 247 247 F F F 13 260-279 120 120 120 244 244 244 F F F 14 280-299 123 123 123 249 249 249 F F F 15 300-319 124 124 124 253 253 253 F F F 16 320-339 123 123 123 249 249 249 F F F 17 340-359 121 121 121 246 246 246 F F F 18 360-379 124 124 124 251 251 251 F F F 19 380-399 125 125 125 254 254 254 F F F 20 400-419 125 125 125 254 254 254 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 125 125 125 254 254 254 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 124 124 124 253 253 253 F F F 25 500-519 124 124 124 252 252 252 F F F 26 520-539 120 120 120 245 245 245 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 121 121 121 246 246 246 F F F Max gen: 125, Max flush: 254 Begin loop 63 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.39557264Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396068036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396110756Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686} {"msg":"Crucible f00ec4db-dea5-4c81-9399-00e5d99a177c has session id: ac4af541-7e0e-4dfa-b3f5-db4684741a27","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.39640385Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Upstairs opts: Upstairs UUID: f00ec4db-dea5-4c81-9399-00e5d99a177c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396449041Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Crucible stats registered with UUID: f00ec4db-dea5-4c81-9399-00e5d99a177c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396472521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396744625Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686} {"msg":"f00ec4db-dea5-4c81-9399-00e5d99a177c active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396886066Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396903807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396923257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396947027Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396975447Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.396997928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.397020498Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.397048979Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.397074289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.397093829Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_connection connected from Ok(127.0.0.1:57723)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.397312162Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_connection connected from Ok(127.0.0.1:41673)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.397351182Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:51:44.397629586Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:51:44.397659206Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:51:44.397677196Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.397690647Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.397704187Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.397793148Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.398167263Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.398193503Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.398208093Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.398224963Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.398834971Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.398956593Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.399344148Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.399370698Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.399385958Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.399398658Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.399412569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:44.399425979Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.398355335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_connection connected from Ok(127.0.0.1:38539)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.398511587Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.399765123Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.399806094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400281689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400828036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400851567Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[0]R flush_numbers[0..12]: [246, 250, 254, 254, 249, 248, 247, 253, 253, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400866977Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[0]R generation[0..12]: [121, 124, 125, 125, 123, 122, 122, 124, 124, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400881567Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400896087Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[1]R flush_numbers[0..12]: [246, 250, 254, 254, 249, 248, 247, 253, 253, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400910068Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[1]R generation[0..12]: [121, 124, 125, 125, 123, 122, 122, 124, 124, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400923778Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400937908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[2]R flush_numbers[0..12]: [246, 250, 254, 254, 249, 248, 247, 253, 253, 231, 240, 237]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400951998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[2]R generation[0..12]: [121, 124, 125, 125, 123, 122, 122, 124, 124, 114, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400965998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.400980048Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:51:54.400993489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.401030189Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.401045949Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.401059059Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"":"downstairs","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.401072379Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.40108744Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"0","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.40111165Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"1","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.40113321Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"client":"2","":"downstairs","session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.4011536Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.401176761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"f00ec4db-dea5-4c81-9399-00e5d99a177c is now active with session: ac4af541-7e0e-4dfa-b3f5-db4684741a27","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.401199861Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.401220901Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686,"session_id":"ac4af541-7e0e-4dfa-b3f5-db4684741a27"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:51:54.401242762Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 223 len 8192 data: 11 13 02/30 Write block 586 len 16384 data: 10 9 11 11 03/30 Write block 13 len 40960 data: 8 8 7 9 9 12 12 12 11 10 04/30 Flush 05/30 Write block 43 len 32768 data: 11 10 9 9 8 7 8 8 06/30 Write block 487 len 32768 data: 10 10 9 10 12 12 10 10 07/30 Write block 493 len 28672 data: 11 11 9 10 11 12 12 08/30 Read block 196 len 40960 09/30 Read block 92 len 40960 10/30 Write block 365 len 8192 data: 6 7 11/30 Flush 12/30 Write block 306 len 8192 data: 8 9 13/30 Read block 385 len 28672 14/30 Write block 76 len 8192 data: 10 11 15/30 Read block 374 len 40960 16/30 Write block 272 len 16384 data: 6 6 6 7 17/30 Write block 389 len 36864 data: 9 10 10 10 12 11 11 13 14 18/30 Read block 200 len 8192 19/30 Read block 401 len 8192 20/30 Write block 239 len 20480 data: 11 9 11 10 11 21/30 Read block 68 len 24576 22/30 Read block 44 len 24576 23/30 Write block 524 len 36864 data: 8 9 15 15 16 16 14 15 13 24/30 Write block 29 len 32768 data: 11 7 8 6 6 6 6 6 25/30 Read block 553 len 16384 26/30 Read block 107 len 20480 27/30 Read block 487 len 40960 28/30 Write block 451 len 24576 data: 13 14 13 9 9 8 29/30 Write block 513 len 32768 data: 4 5 5 6 7 7 6 6 30/30 Read block 578 len 40960 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 4 Done Done Done false 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Write 6 Done Done Done false 30 Acked 1029 Write 8 Done Done Done false 31 Acked 1030 Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:51:54.429385969Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 255 255 255 F F F 1 020-039 126 126 126 255 255 255 T T T 2 040-059 126 126 126 256 256 256 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 123 123 123 249 249 249 F F F 5 100-119 122 122 122 248 248 248 F F F 6 120-139 122 122 122 247 247 247 F F F 7 140-159 124 124 124 253 253 253 F F F 8 160-179 124 124 124 253 253 253 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 126 126 126 255 255 255 T T T 12 240-259 122 122 122 247 247 247 T T T 13 260-279 120 120 120 244 244 244 T T T 14 280-299 123 123 123 249 249 249 F F F 15 300-319 126 126 126 257 257 257 F F F 16 320-339 123 123 123 249 249 249 F F F 17 340-359 121 121 121 246 246 246 F F F 18 360-379 126 126 126 256 256 256 F F F 19 380-399 125 125 125 254 254 254 T T T 20 400-419 125 125 125 254 254 254 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 125 125 125 254 254 254 T T T 23 460-479 120 120 120 244 244 244 F F F 24 480-499 126 126 126 256 256 256 F F F 25 500-519 124 124 124 252 252 252 T T T 26 520-539 120 120 120 245 245 245 T T T 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 126 126 126 255 255 255 F F F Max gen: 126, Max flush: 257 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 255 255 255 F F F 1 020-039 127 127 127 258 258 258 F F F 2 040-059 126 126 126 256 256 256 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 123 123 123 249 249 249 F F F 5 100-119 122 122 122 248 248 248 F F F 6 120-139 122 122 122 247 247 247 F F F 7 140-159 124 124 124 253 253 253 F F F 8 160-179 124 124 124 253 253 253 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 F F F 11 220-239 127 127 127 258 258 258 F F F 12 240-259 127 127 127 258 258 258 F F F 13 260-279 127 127 127 258 258 258 F F F 14 280-299 123 123 123 249 249 249 F F F 15 300-319 126 126 126 257 257 257 F F F 16 320-339 123 123 123 249 249 249 F F F 17 340-359 121 121 121 246 246 246 F F F 18 360-379 126 126 126 256 256 256 F F F 19 380-399 127 127 127 258 258 258 F F F 20 400-419 125 125 125 254 254 254 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 127 127 127 258 258 258 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 126 126 126 256 256 256 F F F 25 500-519 127 127 127 258 258 258 F F F 26 520-539 127 127 127 258 258 258 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 126 126 126 255 255 255 F F F Max gen: 127, Max flush: 258 Begin loop 64 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.696858953Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.697217338Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.697244828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694} {"msg":"Crucible 701367af-3f6a-4439-9f23-7e1f4c9b645e has session id: 8b5db528-aecd-4693-bbcc-c31b7c00cc74","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.697667123Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Upstairs opts: Upstairs UUID: 701367af-3f6a-4439-9f23-7e1f4c9b645e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.697714384Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Crucible stats registered with UUID: 701367af-3f6a-4439-9f23-7e1f4c9b645e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.697738654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.697998287Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694} {"msg":"701367af-3f6a-4439-9f23-7e1f4c9b645e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698091339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698117909Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.69814442Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.69816921Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.69819207Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.69821458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698239351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698287031Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698340932Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698377642Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_connection connected from Ok(127.0.0.1:34841)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698584595Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:08.698613845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:08.698630635Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_connection connected from Ok(127.0.0.1:60559)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698644876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:08.698660606Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698686806Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698712067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.698742367Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.699311894Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.699339004Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.699354055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.699368145Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.699970632Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.700074554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.700461569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.700498949Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.70052803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.70055705Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.70058449Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:08.700606941Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.699530747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_connection connected from Ok(127.0.0.1:56683)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.69972662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.701178058Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.701212768Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.701748265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702267812Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702287672Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[0]R flush_numbers[0..12]: [255, 258, 256, 257, 249, 248, 247, 253, 253, 231, 240, 258]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702302912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[0]R generation[0..12]: [126, 127, 126, 126, 123, 122, 122, 124, 124, 114, 118, 127]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702317582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702332173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[1]R flush_numbers[0..12]: [255, 258, 256, 257, 249, 248, 247, 253, 253, 231, 240, 258]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702346313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[1]R generation[0..12]: [126, 127, 126, 126, 123, 122, 122, 124, 124, 114, 118, 127]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702360203Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702374623Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[2]R flush_numbers[0..12]: [255, 258, 256, 257, 249, 248, 247, 253, 253, 231, 240, 258]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702388554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[2]R generation[0..12]: [126, 127, 126, 126, 123, 122, 122, 124, 124, 114, 118, 127]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702402524Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702421424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:52:18.702435074Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702448654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702461724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702474924Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"":"downstairs","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702488075Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702500105Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"0","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702513515Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"1","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702526735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"client":"2","":"downstairs","session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702540005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702552356Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"701367af-3f6a-4439-9f23-7e1f4c9b645e is now active with session: 8b5db528-aecd-4693-bbcc-c31b7c00cc74","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702565336Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702577366Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694,"session_id":"8b5db528-aecd-4693-bbcc-c31b7c00cc74"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:18.702589636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 243 len 20480 data: 12 13 11 12 12 02/30 Write block 116 len 28672 data: 9 8 9 9 10 8 8 03/30 Read block 7 len 12288 04/30 Read block 261 len 24576 05/30 Write block 221 len 40960 data: 12 13 12 14 12 13 14 13 12 9 06/30 Flush 07/30 Write block 133 len 16384 data: 12 14 13 12 08/30 Read block 369 len 8192 09/30 Read block 173 len 4096 10/30 Read block 415 len 12288 11/30 Flush 12/30 Write block 240 len 20480 data: 10 12 11 13 14 13/30 Read block 446 len 24576 14/30 Write block 445 len 4096 data: 10 15/30 Read block 391 len 32768 16/30 Flush 17/30 Write block 412 len 8192 data: 5 7 18/30 Read block 206 len 16384 19/30 Read block 84 len 8192 20/30 Flush 21/30 Write block 201 len 28672 data: 8 7 8 8 6 6 7 22/30 Read block 216 len 28672 23/30 Read block 381 len 36864 24/30 Read block 77 len 24576 25/30 Read block 149 len 28672 26/30 Write block 210 len 24576 data: 11 11 10 9 8 7 27/30 Read block 132 len 8192 28/30 Write block 48 len 20480 data: 8 9 9 10 13 29/30 Write block 39 len 24576 data: 7 6 8 10 12 11 30/30 Read block 585 len 12288 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 6 Done Done Done false 30 Acked 1029 Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 1 1 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:18.730433979Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 255 255 255 F F F 1 020-039 127 127 127 258 258 258 T T T 2 040-059 126 126 126 256 256 256 T T T 3 060-079 126 126 126 257 257 257 F F F 4 080-099 123 123 123 249 249 249 F F F 5 100-119 128 128 128 259 259 259 F F F 6 120-139 128 128 128 260 260 260 F F F 7 140-159 124 124 124 253 253 253 F F F 8 160-179 124 124 124 253 253 253 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 118 118 118 240 240 240 T T T 11 220-239 128 128 128 259 259 259 F F F 12 240-259 128 128 128 261 261 261 F F F 13 260-279 127 127 127 258 258 258 F F F 14 280-299 123 123 123 249 249 249 F F F 15 300-319 126 126 126 257 257 257 F F F 16 320-339 123 123 123 249 249 249 F F F 17 340-359 121 121 121 246 246 246 F F F 18 360-379 126 126 126 256 256 256 F F F 19 380-399 127 127 127 258 258 258 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 128 128 128 261 261 261 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 126 126 126 256 256 256 F F F 25 500-519 127 127 127 258 258 258 F F F 26 520-539 127 127 127 258 258 258 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 126 126 126 255 255 255 F F F Max gen: 128, Max flush: 262 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 255 255 255 F F F 1 020-039 129 129 129 263 263 263 F F F 2 040-059 129 129 129 263 263 263 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 123 123 123 249 249 249 F F F 5 100-119 128 128 128 259 259 259 F F F 6 120-139 128 128 128 260 260 260 F F F 7 140-159 124 124 124 253 253 253 F F F 8 160-179 124 124 124 253 253 253 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 129 129 129 263 263 263 F F F 11 220-239 128 128 128 259 259 259 F F F 12 240-259 128 128 128 261 261 261 F F F 13 260-279 127 127 127 258 258 258 F F F 14 280-299 123 123 123 249 249 249 F F F 15 300-319 126 126 126 257 257 257 F F F 16 320-339 123 123 123 249 249 249 F F F 17 340-359 121 121 121 246 246 246 F F F 18 360-379 126 126 126 256 256 256 F F F 19 380-399 127 127 127 258 258 258 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 128 128 128 261 261 261 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 126 126 126 256 256 256 F F F 25 500-519 127 127 127 258 258 258 F F F 26 520-539 127 127 127 258 258 258 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 126 126 126 255 255 255 F F F Max gen: 129, Max flush: 263 Begin loop 65 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.938898054Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.939234448Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.939253699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702} {"msg":"Crucible 8137481c-c077-48f0-a162-a99022718045 has session id: 5217146f-9993-4ebd-acf1-dcd140538088","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.939698854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Upstairs opts: Upstairs UUID: 8137481c-c077-48f0-a162-a99022718045, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.939742945Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Crucible stats registered with UUID: 8137481c-c077-48f0-a162-a99022718045","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.939766255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940065369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702} {"msg":"8137481c-c077-48f0-a162-a99022718045 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940094339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940109359Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.94012422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.94013818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.94015195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.94017469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940199821Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940230341Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940256421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940278262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_connection connected from Ok(127.0.0.1:36673)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940548665Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_connection connected from Ok(127.0.0.1:54620)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940696287Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:32.940736607Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:32.940763648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:32.940839059Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940861909Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.940877259Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.94097839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.941510717Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.941536608Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.941551078Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.941564998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.942167176Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.942195326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.942719383Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.942757503Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.942786764Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.942805664Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.942820114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:32.942833394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.942066955Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_connection connected from Ok(127.0.0.1:51894)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.942195976Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.94332202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.943361721Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.943931678Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944481785Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944515326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[0]R flush_numbers[0..12]: [255, 263, 263, 257, 249, 259, 260, 253, 253, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944535916Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[0]R generation[0..12]: [126, 129, 129, 126, 123, 128, 128, 124, 124, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944554696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944573326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[1]R flush_numbers[0..12]: [255, 263, 263, 257, 249, 259, 260, 253, 253, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944591377Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[1]R generation[0..12]: [126, 129, 129, 126, 123, 128, 128, 124, 124, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944608927Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944626847Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[2]R flush_numbers[0..12]: [255, 263, 263, 257, 249, 259, 260, 253, 253, 231, 263, 259]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:52:42.944644157Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[2]R generation[0..12]: [126, 129, 129, 126, 123, 128, 128, 124, 124, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944670428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944688778Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944705378Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944722038Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944738389Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944754589Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"":"downstairs","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944771869Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944787009Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"0","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.944804309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"1","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.94482133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"client":"2","":"downstairs","session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.94483852Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.94485409Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"8137481c-c077-48f0-a162-a99022718045 is now active with session: 5217146f-9993-4ebd-acf1-dcd140538088","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.94487018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "time":"2024-07-10T15:52:42.94488517Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702,"session_id":"5217146f-9993-4ebd-acf1-dcd140538088"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:42.9449048Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 317 len 28672 data: 8 9 11 10 10 13 13 02/30 Read block 393 len 16384 03/30 Read block 94 len 28672 04/30 Write block 173 len 16384 data: 9 10 8 6 05/30 Write block 116 len 20480 data: 10 9 10 10 11 06/30 Read block 342 len 4096 07/30 Read block 550 len 8192 08/30 Flush 09/30 Write block 301 len 24576 data: 6 8 9 9 9 9 10/30 Write block 40 len 8192 data: 7 9 11/30 Write block 340 len 12288 data: 11 12 13 12/30 Write block 273 len 40960 data: 7 7 8 6 5 8 7 6 7 7 13/30 Read block 556 len 36864 14/30 Read block 506 len 32768 15/30 Write block 537 len 12288 data: 11 13 10 16/30 Read block 359 len 24576 17/30 Read block 343 len 32768 18/30 Write block 242 len 20480 data: 12 14 15 12 13 19/30 Read block 317 len 24576 20/30 Write block 290 len 36864 data: 7 8 10 11 10 7 8 8 7 21/30 Read block 182 len 32768 22/30 Read block 8 len 8192 23/30 Write block 105 len 28672 data: 8 8 7 8 8 7 7 24/30 Read block 472 len 4096 25/30 Write block 161 len 16384 data: 8 7 5 6 26/30 Read block 558 len 28672 27/30 Write block 156 len 12288 data: 11 10 13 28/30 Write block 24 len 28672 data: 8 9 9 9 10 12 8 29/30 Read block 468 len 24576 30/30 Write block 371 len 4096 data: 13 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 7 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:42.970612107Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 255 255 255 F F F 1 020-039 129 129 129 263 263 263 T T T 2 040-059 130 130 130 265 265 265 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 123 123 123 249 249 249 F F F 5 100-119 130 130 130 264 264 264 T T T 6 120-139 130 130 130 264 264 264 F F F 7 140-159 124 124 124 253 253 253 T T T 8 160-179 130 130 130 264 264 264 T T T 9 180-199 114 114 114 231 231 231 F F F 10 200-219 129 129 129 263 263 263 F F F 11 220-239 128 128 128 259 259 259 F F F 12 240-259 130 130 130 265 265 265 F F F 13 260-279 130 130 130 265 265 265 F F F 14 280-299 130 130 130 265 265 265 F F F 15 300-319 130 130 130 265 265 265 F F F 16 320-339 130 130 130 264 264 264 F F F 17 340-359 130 130 130 265 265 265 F F F 18 360-379 126 126 126 256 256 256 T T T 19 380-399 127 127 127 258 258 258 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 128 128 128 261 261 261 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 126 126 126 256 256 256 F F F 25 500-519 127 127 127 258 258 258 F F F 26 520-539 130 130 130 265 265 265 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 126 126 126 255 255 255 F F F Max gen: 130, Max flush: 265 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 255 255 255 F F F 1 020-039 131 131 131 266 266 266 F F F 2 040-059 130 130 130 265 265 265 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 123 123 123 249 249 249 F F F 5 100-119 131 131 131 266 266 266 F F F 6 120-139 130 130 130 264 264 264 F F F 7 140-159 131 131 131 266 266 266 F F F 8 160-179 131 131 131 266 266 266 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 129 129 129 263 263 263 F F F 11 220-239 128 128 128 259 259 259 F F F 12 240-259 130 130 130 265 265 265 F F F 13 260-279 130 130 130 265 265 265 F F F 14 280-299 130 130 130 265 265 265 F F F 15 300-319 130 130 130 265 265 265 F F F 16 320-339 130 130 130 264 264 264 F F F 17 340-359 130 130 130 265 265 265 F F F 18 360-379 131 131 131 266 266 266 F F F 19 380-399 127 127 127 258 258 258 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 128 128 128 261 261 261 F F F 23 460-479 120 120 120 244 244 244 F F F 24 480-499 126 126 126 256 256 256 F F F 25 500-519 127 127 127 258 258 258 F F F 26 520-539 130 130 130 265 265 265 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 126 126 126 255 255 255 F F F Max gen: 131, Max flush: 266 Begin loop 66 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.200647566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.201118992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.201146862Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710} {"msg":"Crucible ad77cbb0-c65a-4a1d-b79e-8c1f48d552b0 has session id: cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.201541907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Upstairs opts: Upstairs UUID: ad77cbb0-c65a-4a1d-b79e-8c1f48d552b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.201590368Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Crucible stats registered with UUID: ad77cbb0-c65a-4a1d-b79e-8c1f48d552b0","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.201618908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.201898142Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710} {"msg":"ad77cbb0-c65a-4a1d-b79e-8c1f48d552b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202001173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202027193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202043123Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202057134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202070864Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202084684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202098564Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202116255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202130405Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202144165Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:57.202471999Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:57.20251718Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54098)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.20254753Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:52:57.20257766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202604351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202626511Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37175)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202651851Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.202671831Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.203102897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.203127317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.203141408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.203154918Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.203757965Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.203784306Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.204287772Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.204314622Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.204330263Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.204356223Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.204370513Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:52:57.204388703Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.203598394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_connection connected from Ok(127.0.0.1:65019)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.204017249Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.205093333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.205120053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.20566345Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206196436Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206219647Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[0]R flush_numbers[0..12]: [255, 266, 265, 257, 249, 266, 264, 266, 266, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206234397Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[0]R generation[0..12]: [126, 131, 130, 126, 123, 131, 130, 131, 131, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206250037Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206264747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[1]R flush_numbers[0..12]: [255, 266, 265, 257, 249, 266, 264, 266, 266, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206278857Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[1]R generation[0..12]: [126, 131, 130, 126, 123, 131, 130, 131, 131, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206292938Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206307298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[2]R flush_numbers[0..12]: [255, 266, 265, 257, 249, 266, 264, 266, 266, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206326028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[2]R generation[0..12]: [126, 131, 130, 126, 123, 131, 130, 131, 131, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206340578Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206354829Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Max found gen is 132","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-07-10T15:53:07.206371639Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206385429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206398759Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206411989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"":"downstairs","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206425419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.20643731Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"0","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.20645082Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"1","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.20646424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"client":"2","":"downstairs","session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.20647758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.20648998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"ad77cbb0-c65a-4a1d-b79e-8c1f48d552b0 is now active with session: cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.20650333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206515631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710,"session_id":"cdd0e8bc-c527-42a0-bc15-36dd3b6d0fe9"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:07.206528121Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 99 len 36864 data: 10 12 10 8 12 10 9 9 8 02/30 Read block 238 len 28672 03/30 Read block 235 len 20480 04/30 Write block 5 len 28672 data: 12 11 10 11 9 8 9 05/30 Write block 464 len 16384 data: 8 8 7 6 06/30 Read block 101 len 36864 07/30 Read block 422 len 28672 08/30 Read block 378 len 40960 09/30 Write block 508 len 4096 data: 6 10/30 Read block 479 len 20480 11/30 Read block 257 len 40960 12/30 Write block 508 len 8192 data: 7 8 13/30 Write block 483 len 12288 data: 9 9 8 14/30 Read block 294 len 32768 15/30 Read block 313 len 20480 16/30 Read block 210 len 32768 17/30 Write block 300 len 32768 data: 6 7 9 10 10 10 10 10 18/30 Write block 107 len 32768 data: 9 9 9 8 8 7 9 8 19/30 Write block 319 len 36864 data: 12 11 11 14 14 13 12 13 12 20/30 Flush 21/30 Write block 128 len 16384 data: 7 9 16 15 22/30 Read block 303 len 12288 23/30 Read block 83 len 16384 24/30 Write block 386 len 20480 data: 11 8 8 10 11 25/30 Flush 26/30 Write block 587 len 12288 data: 10 12 12 27/30 Write block 244 len 28672 data: 16 13 14 13 14 13 13 28/30 Read block 424 len 4096 29/30 Write block 262 len 12288 data: 10 8 8 30/30 Read block 503 len 12288 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:53:07.234375084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 267 267 267 F F F 1 020-039 131 131 131 266 266 266 F F F 2 040-059 130 130 130 265 265 265 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 132 132 132 267 267 267 F F F 5 100-119 132 132 132 267 267 267 F F F 6 120-139 132 132 132 268 268 268 F F F 7 140-159 131 131 131 266 266 266 F F F 8 160-179 131 131 131 266 266 266 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 129 129 129 263 263 263 F F F 11 220-239 128 128 128 259 259 259 F F F 12 240-259 130 130 130 265 265 265 T T T 13 260-279 130 130 130 265 265 265 T T T 14 280-299 130 130 130 265 265 265 F F F 15 300-319 132 132 132 267 267 267 F F F 16 320-339 132 132 132 267 267 267 F F F 17 340-359 130 130 130 265 265 265 F F F 18 360-379 131 131 131 266 266 266 F F F 19 380-399 132 132 132 268 268 268 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 128 128 128 261 261 261 F F F 23 460-479 132 132 132 267 267 267 F F F 24 480-499 132 132 132 267 267 267 F F F 25 500-519 132 132 132 267 267 267 F F F 26 520-539 130 130 130 265 265 265 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 126 126 126 255 255 255 T T T Max gen: 132, Max flush: 268 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 267 267 267 F F F 1 020-039 131 131 131 266 266 266 F F F 2 040-059 130 130 130 265 265 265 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 132 132 132 267 267 267 F F F 5 100-119 132 132 132 267 267 267 F F F 6 120-139 132 132 132 268 268 268 F F F 7 140-159 131 131 131 266 266 266 F F F 8 160-179 131 131 131 266 266 266 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 129 129 129 263 263 263 F F F 11 220-239 128 128 128 259 259 259 F F F 12 240-259 133 133 133 269 269 269 F F F 13 260-279 133 133 133 269 269 269 F F F 14 280-299 130 130 130 265 265 265 F F F 15 300-319 132 132 132 267 267 267 F F F 16 320-339 132 132 132 267 267 267 F F F 17 340-359 130 130 130 265 265 265 F F F 18 360-379 131 131 131 266 266 266 F F F 19 380-399 132 132 132 268 268 268 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 119 119 119 242 242 242 F F F 22 440-459 128 128 128 261 261 261 F F F 23 460-479 132 132 132 267 267 267 F F F 24 480-499 132 132 132 267 267 267 F F F 25 500-519 132 132 132 267 267 267 F F F 26 520-539 130 130 130 265 265 265 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 123 123 123 249 249 249 F F F 29 580-599 133 133 133 269 269 269 F F F Max gen: 133, Max flush: 269 Begin loop 67 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.436308557Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.43653831Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.43655429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718} {"msg":"Crucible 6bc2d27f-9066-4462-93d7-ebe33bcdf9be has session id: 76bd1dc4-8273-4a5f-a419-cf0344df39fb","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437130627Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Upstairs opts: Upstairs UUID: 6bc2d27f-9066-4462-93d7-ebe33bcdf9be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437174758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Crucible stats registered with UUID: 6bc2d27f-9066-4462-93d7-ebe33bcdf9be","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437196468Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437497822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718} {"msg":"6bc2d27f-9066-4462-93d7-ebe33bcdf9be active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437519722Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437540242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437565563Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437589693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437611553Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437634104Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437659584Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437684954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437769815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437797996Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:53:21.437825416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:53:21.437853086Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:53:21.437907927Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51503)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437935547Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437959538Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.437978298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:64935)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.43810925Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.438212181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.438702037Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.438727737Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.438742398Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.438756078Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.439380606Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.439408036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.439918602Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.439944663Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.439959033Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.440027144Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.440046164Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:21.440064594Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.438632646Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:36790)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.438856659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.439959133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.439988193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.44052687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441093287Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441120268Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[0]R flush_numbers[0..12]: [267, 266, 265, 257, 267, 267, 268, 266, 266, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441138688Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[0]R generation[0..12]: [132, 131, 130, 126, 132, 132, 132, 131, 131, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441156978Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441174948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[1]R flush_numbers[0..12]: [267, 266, 265, 257, 267, 267, 268, 266, 266, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441192798Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[1]R generation[0..12]: [132, 131, 130, 126, 132, 132, 132, 131, 131, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441210339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441228199Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[2]R flush_numbers[0..12]: [267, 266, 265, 257, 267, 267, 268, 266, 266, 231, 263, 259]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:53:31.441263599Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[2]R generation[0..12]: [132, 131, 130, 126, 132, 132, 132, 131, 131, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.44128311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.44131766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.44133435Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441350991Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441367541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"":"downstairs","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441384191Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441399411Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"0","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441416261Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"1","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441432721Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"client":"2","":"downstairs","session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441449462Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441464872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"6bc2d27f-9066-4462-93d7-ebe33bcdf9be is now active with session: 76bd1dc4-8273-4a5f-a419-cf0344df39fb","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-07-10T15:53:31.441481052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441497632Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718,"session_id":"76bd1dc4-8273-4a5f-a419-cf0344df39fb"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:31.441515893Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 574 len 20480 data: 9 9 12 11 13 02/30 Read block 369 len 32768 03/30 Flush 04/30 Write block 530 len 28672 data: 15 16 14 8 9 7 9 05/30 Read block 338 len 40960 06/30 Read block 370 len 12288 07/30 Read block 28 len 4096 08/30 Read block 492 len 8192 09/30 Write block 165 len 40960 data: 7 9 10 9 8 7 6 7 10 11 10/30 Write block 287 len 16384 data: 8 8 9 8 11/30 Write block 86 len 36864 data: 7 10 9 9 9 9 9 10 8 12/30 Read block 370 len 8192 13/30 Write block 566 len 12288 data: 10 11 10 14/30 Write block 305 len 24576 data: 11 11 11 9 8 8 15/30 Read block 437 len 32768 16/30 Read block 510 len 8192 17/30 Write block 282 len 16384 data: 8 6 5 6 18/30 Read block 142 len 12288 19/30 Read block 101 len 32768 20/30 Read block 502 len 8192 21/30 Write block 173 len 24576 data: 11 12 9 7 8 7 22/30 Read block 24 len 36864 23/30 Flush 24/30 Write block 135 len 4096 data: 14 25/30 Read block 171 len 12288 26/30 Read block 575 len 16384 27/30 Read block 389 len 40960 28/30 Read block 0 len 40960 29/30 Read block 578 len 20480 30/30 Write block 425 len 32768 data: 10 10 11 11 12 10 10 9 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:53:31.467266679Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 267 267 267 F F F 1 020-039 131 131 131 266 266 266 F F F 2 040-059 130 130 130 265 265 265 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 134 134 134 271 271 271 F F F 5 100-119 132 132 132 267 267 267 F F F 6 120-139 132 132 132 268 268 268 T T T 7 140-159 131 131 131 266 266 266 F F F 8 160-179 134 134 134 271 271 271 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 129 129 129 263 263 263 F F F 11 220-239 128 128 128 259 259 259 F F F 12 240-259 133 133 133 269 269 269 F F F 13 260-279 133 133 133 269 269 269 F F F 14 280-299 134 134 134 271 271 271 F F F 15 300-319 134 134 134 271 271 271 F F F 16 320-339 132 132 132 267 267 267 F F F 17 340-359 130 130 130 265 265 265 F F F 18 360-379 131 131 131 266 266 266 F F F 19 380-399 132 132 132 268 268 268 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 119 119 119 242 242 242 T T T 22 440-459 128 128 128 261 261 261 F F F 23 460-479 132 132 132 267 267 267 F F F 24 480-499 132 132 132 267 267 267 F F F 25 500-519 132 132 132 267 267 267 F F F 26 520-539 134 134 134 271 271 271 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 134 134 134 271 271 271 F F F 29 580-599 133 133 133 269 269 269 F F F Max gen: 134, Max flush: 271 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 267 267 267 F F F 1 020-039 131 131 131 266 266 266 F F F 2 040-059 130 130 130 265 265 265 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 134 134 134 271 271 271 F F F 5 100-119 132 132 132 267 267 267 F F F 6 120-139 135 135 135 272 272 272 F F F 7 140-159 131 131 131 266 266 266 F F F 8 160-179 134 134 134 271 271 271 F F F 9 180-199 114 114 114 231 231 231 F F F 10 200-219 129 129 129 263 263 263 F F F 11 220-239 128 128 128 259 259 259 F F F 12 240-259 133 133 133 269 269 269 F F F 13 260-279 133 133 133 269 269 269 F F F 14 280-299 134 134 134 271 271 271 F F F 15 300-319 134 134 134 271 271 271 F F F 16 320-339 132 132 132 267 267 267 F F F 17 340-359 130 130 130 265 265 265 F F F 18 360-379 131 131 131 266 266 266 F F F 19 380-399 132 132 132 268 268 268 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 135 135 135 272 272 272 F F F 22 440-459 128 128 128 261 261 261 F F F 23 460-479 132 132 132 267 267 267 F F F 24 480-499 132 132 132 267 267 267 F F F 25 500-519 132 132 132 267 267 267 F F F 26 520-539 134 134 134 271 271 271 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 134 134 134 271 271 271 F F F 29 580-599 133 133 133 269 269 269 F F F Max gen: 135, Max flush: 272 Begin loop 68 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.656732184Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.657038208Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.657056438Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726} {"msg":"Crucible 372df97d-19e8-499c-90a1-0f70dfe72b7f has session id: 540ef279-29f1-4a8f-8715-8cbd16869c9d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.657491793Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Upstairs opts: Upstairs UUID: 372df97d-19e8-499c-90a1-0f70dfe72b7f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.657539544Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Crucible stats registered with UUID: 372df97d-19e8-499c-90a1-0f70dfe72b7f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.657567354Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.657875338Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726} {"msg":"372df97d-19e8-499c-90a1-0f70dfe72b7f active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.657904969Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.657920069Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.657942089Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.6579669Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.65799173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.65803671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.658062461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.658085351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.658100961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.658115451Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34495)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.658390845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49574)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.658429965Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:53:45.658620288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:53:45.658664458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:53:45.6587691Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.65878878Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.65880375Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.658881241Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.659271456Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.659295456Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.659310397Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.659324577Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.659935914Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.659964845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.660500592Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.660530052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.660547882Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.660563532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.660584373Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:45.660598343Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.660239958Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44194)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.66037477Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.661527995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.661555135Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662142233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662663859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662687609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[0]R flush_numbers[0..12]: [267, 266, 265, 257, 271, 267, 272, 266, 271, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662705929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[0]R generation[0..12]: [132, 131, 130, 126, 134, 132, 135, 131, 134, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.66272494Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.6627431Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[1]R flush_numbers[0..12]: [267, 266, 265, 257, 271, 267, 272, 266, 271, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.66276119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[1]R generation[0..12]: [132, 131, 130, 126, 134, 132, 135, 131, 134, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.66277893Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662796751Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[2]R flush_numbers[0..12]: [267, 266, 265, 257, 271, 267, 272, 266, 271, 231, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662819701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [132, 131, 130, 126, 134, 132, 135, 131, 134, 114, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662840261Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662858671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662875202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662891922Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662908462Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662924762Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"":"downstairs","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662941483Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662956623Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"0","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662973543Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"1","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.662990263Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"client":"2","":"downstairs","session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.663007073Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.663022444Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"372df97d-19e8-499c-90a1-0f70dfe72b7f is now active with session: 540ef279-29f1-4a8f-8715-8cbd16869c9d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.663040014Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.663055344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726,"session_id":"540ef279-29f1-4a8f-8715-8cbd16869c9d"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:53:55.663073954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 8 len 4096 data: 12 02/30 Read block 375 len 32768 03/30 Read block 130 len 24576 04/30 Read block 29 len 40960 05/30 Write block 508 len 24576 data: 8 9 7 7 5 5 06/30 Read block 157 len 36864 07/30 Write block 187 len 28672 data: 6 7 8 8 10 12 10 08/30 Read block 104 len 24576 09/30 Read block 463 len 32768 10/30 Read block 399 len 20480 11/30 Write block 196 len 4096 data: 12 12/30 Write block 314 len 16384 data: 8 7 8 9 13/30 Write block 82 len 8192 data: 7 8 14/30 Read block 481 len 4096 15/30 Read block 188 len 24576 16/30 Read block 142 len 32768 17/30 Read block 225 len 36864 18/30 Write block 176 len 24576 data: 8 9 8 8 7 5 19/30 Write block 168 len 8192 data: 10 9 20/30 Read block 280 len 32768 21/30 Read block 388 len 40960 22/30 Write block 3 len 12288 data: 10 11 13 23/30 Read block 345 len 20480 24/30 Write block 379 len 32768 data: 16 14 11 11 6 5 8 12 25/30 Flush 26/30 Write block 137 len 12288 data: 15 17 17 27/30 Read block 92 len 24576 28/30 Write block 430 len 28672 data: 11 11 10 8 5 6 7 29/30 Write block 103 len 24576 data: 13 11 10 10 10 10 30/30 Write block 6 len 8192 data: 12 11 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 7 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false 31 Acked 1030 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:53:55.687114939Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 273 273 273 T T T 1 020-039 131 131 131 266 266 266 F F F 2 040-059 130 130 130 265 265 265 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 136 136 136 273 273 273 F F F 5 100-119 132 132 132 267 267 267 T T T 6 120-139 135 135 135 272 272 272 T T T 7 140-159 131 131 131 266 266 266 F F F 8 160-179 136 136 136 273 273 273 F F F 9 180-199 136 136 136 273 273 273 F F F 10 200-219 129 129 129 263 263 263 F F F 11 220-239 128 128 128 259 259 259 F F F 12 240-259 133 133 133 269 269 269 F F F 13 260-279 133 133 133 269 269 269 F F F 14 280-299 134 134 134 271 271 271 F F F 15 300-319 136 136 136 273 273 273 F F F 16 320-339 132 132 132 267 267 267 F F F 17 340-359 130 130 130 265 265 265 F F F 18 360-379 136 136 136 274 274 274 F F F 19 380-399 136 136 136 274 274 274 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 135 135 135 272 272 272 T T T 22 440-459 128 128 128 261 261 261 F F F 23 460-479 132 132 132 267 267 267 F F F 24 480-499 132 132 132 267 267 267 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 134 134 134 271 271 271 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 134 134 134 271 271 271 F F F 29 580-599 133 133 133 269 269 269 F F F Max gen: 136, Max flush: 274 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 275 275 275 F F F 1 020-039 131 131 131 266 266 266 F F F 2 040-059 130 130 130 265 265 265 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 136 136 136 273 273 273 F F F 5 100-119 137 137 137 275 275 275 F F F 6 120-139 137 137 137 275 275 275 F F F 7 140-159 131 131 131 266 266 266 F F F 8 160-179 136 136 136 273 273 273 F F F 9 180-199 136 136 136 273 273 273 F F F 10 200-219 129 129 129 263 263 263 F F F 11 220-239 128 128 128 259 259 259 F F F 12 240-259 133 133 133 269 269 269 F F F 13 260-279 133 133 133 269 269 269 F F F 14 280-299 134 134 134 271 271 271 F F F 15 300-319 136 136 136 273 273 273 F F F 16 320-339 132 132 132 267 267 267 F F F 17 340-359 130 130 130 265 265 265 F F F 18 360-379 136 136 136 274 274 274 F F F 19 380-399 136 136 136 274 274 274 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 137 137 137 275 275 275 F F F 22 440-459 128 128 128 261 261 261 F F F 23 460-479 132 132 132 267 267 267 F F F 24 480-499 132 132 132 267 267 267 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 134 134 134 271 271 271 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 134 134 134 271 271 271 F F F 29 580-599 133 133 133 269 269 269 F F F Max gen: 137, Max flush: 275 Begin loop 69 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.900988683Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.901254776Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.901271097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734} {"msg":"Crucible d38d25ae-2472-47ff-ba0a-5d233be25dc5 has session id: a2e487fd-b014-4f59-bcaa-ff8945eeadfb","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.901702262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Upstairs opts: Upstairs UUID: d38d25ae-2472-47ff-ba0a-5d233be25dc5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.901749743Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Crucible stats registered with UUID: d38d25ae-2472-47ff-ba0a-5d233be25dc5","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.901774273Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902103207Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734} {"msg":"d38d25ae-2472-47ff-ba0a-5d233be25dc5 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902176478Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902194978Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902209968Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902223859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902237569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902251659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902271489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.90230025Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.90231466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.90232852Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:09.902486962Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:09.902526273Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:09.902553623Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_connection connected from Ok(127.0.0.1:39803)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902578723Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902604493Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_connection connected from Ok(127.0.0.1:37796)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902669614Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902692845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.902809326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.903379463Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.903406604Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.903421324Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.903435494Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.904010671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.904130753Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.904529818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.904556968Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.904571758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.904595459Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.904613629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:09.904627639Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.904196224Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_connection connected from Ok(127.0.0.1:46716)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.904352246Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.90546169Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.90549573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906120668Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906715146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906733826Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[0]R flush_numbers[0..12]: [275, 266, 265, 257, 273, 275, 275, 266, 273, 273, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906748626Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[0]R generation[0..12]: [137, 131, 130, 126, 136, 137, 137, 131, 136, 136, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906763456Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906777766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[1]R flush_numbers[0..12]: [275, 266, 265, 257, 273, 275, 275, 266, 273, 273, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906791907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[1]R generation[0..12]: [137, 131, 130, 126, 136, 137, 137, 131, 136, 136, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906805987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906820247Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[2]R flush_numbers[0..12]: [275, 266, 265, 257, 273, 275, 275, 266, 273, 273, 263, 259]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906839097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[2]R generation[0..12]: [137, 131, 130, 126, 136, 137, 137, 131, 136, 136, 129, 128]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906853487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906867518Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906880588Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906897388Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906910518Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906923428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"":"downstairs","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906936659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906948719Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"0","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906962089Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"1","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906976119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"client":"2","":"downstairs","session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.906989459Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.907009269Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"d38d25ae-2472-47ff-ba0a-5d233be25dc5 is now active with session: a2e487fd-b014-4f59-bcaa-ff8945eeadfb","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.9070236Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.90703604Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734,"session_id":"a2e487fd-b014-4f59-bcaa-ff8945eeadfb"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:19.90705114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 469 len 36864 data: 5 6 5 5 8 8 12 16 15 02/30 Read block 553 len 36864 03/30 Read block 213 len 16384 04/30 Read block 351 len 28672 05/30 Write block 464 len 32768 data: 9 9 8 7 7 6 7 6 06/30 Write block 238 len 20480 data: 11 12 11 13 13 07/30 Read block 196 len 8192 08/30 Write block 484 len 12288 data: 10 9 8 09/30 Write block 53 len 28672 data: 10 9 9 12 10 10 11 10/30 Write block 200 len 4096 data: 10 11/30 Write block 273 len 32768 data: 8 8 9 7 6 9 8 7 12/30 Read block 404 len 40960 13/30 Write block 2 len 24576 data: 9 11 12 14 13 12 14/30 Read block 354 len 36864 15/30 Flush 16/30 Write block 131 len 36864 data: 16 14 13 15 15 13 16 18 18 17/30 Write block 241 len 12288 data: 14 14 15 18/30 Read block 522 len 36864 19/30 Write block 339 len 20480 data: 11 12 13 14 13 20/30 Flush 21/30 Write block 469 len 28672 data: 7 8 7 6 9 9 13 22/30 Read block 599 len 4096 23/30 Write block 456 len 40960 data: 9 5 5 4 5 6 7 8 10 10 24/30 Read block 58 len 16384 25/30 Read block 115 len 24576 26/30 Write block 95 len 40960 data: 9 9 9 10 11 13 11 9 14 12 27/30 Read block 14 len 36864 28/30 Write block 39 len 32768 data: 8 8 10 11 13 12 10 10 29/30 Write block 572 len 36864 data: 8 7 10 10 13 12 14 11 10 30/30 Write block 13 len 24576 data: 9 9 8 10 10 13 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 8 Done Done Done false 30 Acked 1029 Write 9 Done Done Done false 31 Acked 1030 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:19.943612684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 277 277 277 T T T 1 020-039 131 131 131 266 266 266 T T T 2 040-059 138 138 138 276 276 276 T T T 3 060-079 126 126 126 257 257 257 F F F 4 080-099 136 136 136 273 273 273 T T T 5 100-119 137 137 137 275 275 275 T T T 6 120-139 138 138 138 278 278 278 F F F 7 140-159 131 131 131 266 266 266 F F F 8 160-179 136 136 136 273 273 273 F F F 9 180-199 136 136 136 273 273 273 F F F 10 200-219 138 138 138 276 276 276 F F F 11 220-239 138 138 138 276 276 276 F F F 12 240-259 138 138 138 278 278 278 F F F 13 260-279 138 138 138 277 277 277 F F F 14 280-299 138 138 138 277 277 277 F F F 15 300-319 136 136 136 273 273 273 F F F 16 320-339 138 138 138 278 278 278 F F F 17 340-359 138 138 138 278 278 278 F F F 18 360-379 136 136 136 274 274 274 F F F 19 380-399 136 136 136 274 274 274 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 137 137 137 275 275 275 F F F 22 440-459 128 128 128 261 261 261 T T T 23 460-479 138 138 138 276 276 276 T T T 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 134 134 134 271 271 271 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 134 134 134 271 271 271 T T T 29 580-599 133 133 133 269 269 269 T T T Max gen: 138, Max flush: 278 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 279 279 279 F F F 1 020-039 139 139 139 279 279 279 F F F 2 040-059 139 139 139 279 279 279 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 139 139 139 279 279 279 F F F 6 120-139 138 138 138 278 278 278 F F F 7 140-159 131 131 131 266 266 266 F F F 8 160-179 136 136 136 273 273 273 F F F 9 180-199 136 136 136 273 273 273 F F F 10 200-219 138 138 138 276 276 276 F F F 11 220-239 138 138 138 276 276 276 F F F 12 240-259 138 138 138 278 278 278 F F F 13 260-279 138 138 138 277 277 277 F F F 14 280-299 138 138 138 277 277 277 F F F 15 300-319 136 136 136 273 273 273 F F F 16 320-339 138 138 138 278 278 278 F F F 17 340-359 138 138 138 278 278 278 F F F 18 360-379 136 136 136 274 274 274 F F F 19 380-399 136 136 136 274 274 274 F F F 20 400-419 128 128 128 262 262 262 F F F 21 420-439 137 137 137 275 275 275 F F F 22 440-459 139 139 139 279 279 279 F F F 23 460-479 139 139 139 279 279 279 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 134 134 134 271 271 271 F F F 27 540-559 120 120 120 245 245 245 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 139 139 139 279 279 279 F F F Max gen: 139, Max flush: 279 Begin loop 70 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.227160822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.227560817Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.227591428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742} {"msg":"Crucible 822123b4-2bb6-45fa-a046-b1c84e5a190c has session id: b0ea1251-5577-4a55-9cfe-bd05cdb87631","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.227961633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Upstairs opts: Upstairs UUID: 822123b4-2bb6-45fa-a046-b1c84e5a190c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228015553Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Crucible stats registered with UUID: 822123b4-2bb6-45fa-a046-b1c84e5a190c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228033874Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228302597Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742} {"msg":"822123b4-2bb6-45fa-a046-b1c84e5a190c active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228426728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228444679Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228464789Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228489459Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228514309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.22853636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.22855974Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.2285848Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228607271Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228630831Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_connection connected from Ok(127.0.0.1:58779)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228830193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:34.228917195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:34.228958855Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_connection connected from Ok(127.0.0.1:60740)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.228989026Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:34.229018046Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.229058387Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.229082487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.229226738Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.229575473Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.229600073Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.229614803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.229628444Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.230226831Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.230358453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.230701047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.230724037Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.230741598Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.230811039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.230829529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:34.230843049Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.229909327Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_connection connected from Ok(127.0.0.1:35799)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.230058599Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.231387146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.231423666Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.231919473Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.23247163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.23249538Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[0]R flush_numbers[0..12]: [279, 279, 279, 257, 279, 279, 278, 266, 273, 273, 276, 276]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.23251335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[0]R generation[0..12]: [139, 139, 139, 126, 139, 139, 138, 131, 136, 136, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.23253112Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232549171Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[1]R flush_numbers[0..12]: [279, 279, 279, 257, 279, 279, 278, 266, 273, 273, 276, 276]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232566771Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[1]R generation[0..12]: [139, 139, 139, 126, 139, 139, 138, 131, 136, 136, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232589541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232607872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[2]R flush_numbers[0..12]: [279, 279, 279, 257, 279, 279, 278, 266, 273, 273, 276, 276]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232625422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R generation[0..12]: [139, 139, 139, 126, 139, 139, 138, 131, 136, 136, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232645702Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232664232Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232681002Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232697783Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232714133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232730463Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"":"downstairs","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232747003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232762014Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"0","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232778804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"1","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232795574Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"client":"2","":"downstairs","session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232812384Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232827884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"822123b4-2bb6-45fa-a046-b1c84e5a190c is now active with session: b0ea1251-5577-4a55-9cfe-bd05cdb87631","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232846824Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"Set Active after no reconciliation","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2024-07-10T15:54:44.232865745Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742,"session_id":"b0ea1251-5577-4a55-9cfe-bd05cdb87631"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:44.232881395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 415 len 40960 data: 3 4 4 8 6 6 6 8 8 10 02/30 Write block 432 len 8192 data: 11 9 03/30 Read block 51 len 32768 04/30 Flush 05/30 Write block 191 len 36864 data: 11 13 11 11 10 13 14 12 10 06/30 Write block 231 len 4096 data: 7 07/30 Read block 488 len 40960 08/30 Write block 163 len 8192 data: 6 7 09/30 Write block 352 len 8192 data: 10 11 10/30 Read block 399 len 32768 11/30 Read block 352 len 20480 12/30 Read block 18 len 4096 13/30 Read block 140 len 12288 14/30 Read block 423 len 16384 15/30 Read block 406 len 28672 16/30 Flush 17/30 Write block 44 len 24576 data: 13 11 11 9 9 10 18/30 Read block 527 len 20480 19/30 Read block 432 len 16384 20/30 Write block 355 len 28672 data: 11 12 10 11 10 9 10 21/30 Flush 22/30 Write block 539 len 16384 data: 11 10 7 8 23/30 Flush 24/30 Write block 423 len 28672 data: 9 11 11 11 12 12 13 25/30 Write block 460 len 32768 data: 6 7 8 9 11 11 9 8 26/30 Read block 542 len 36864 27/30 Write block 7 len 12288 data: 13 13 10 28/30 Read block 109 len 40960 29/30 Write block 183 len 28672 data: 4 4 3 5 7 8 9 30/30 Write block 428 len 32768 data: 13 14 12 12 12 10 6 7 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 7 Done Done Done false 31 Acked 1030 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:44.262690653Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 279 279 279 T T T 1 020-039 139 139 139 279 279 279 F F F 2 040-059 140 140 140 282 282 282 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 139 139 139 279 279 279 F F F 6 120-139 138 138 138 278 278 278 F F F 7 140-159 131 131 131 266 266 266 F F F 8 160-179 140 140 140 281 281 281 F F F 9 180-199 140 140 140 281 281 281 T T T 10 200-219 138 138 138 276 276 276 F F F 11 220-239 140 140 140 281 281 281 F F F 12 240-259 138 138 138 278 278 278 F F F 13 260-279 138 138 138 277 277 277 F F F 14 280-299 138 138 138 277 277 277 F F F 15 300-319 136 136 136 273 273 273 F F F 16 320-339 138 138 138 278 278 278 F F F 17 340-359 140 140 140 283 283 283 F F F 18 360-379 140 140 140 283 283 283 F F F 19 380-399 136 136 136 274 274 274 F F F 20 400-419 140 140 140 280 280 280 F F F 21 420-439 140 140 140 280 280 280 T T T 22 440-459 139 139 139 279 279 279 F F F 23 460-479 139 139 139 279 279 279 T T T 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 140 140 140 284 284 284 F F F 27 540-559 140 140 140 284 284 284 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 139 139 139 279 279 279 F F F Max gen: 140, Max flush: 284 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 285 285 285 F F F 1 020-039 139 139 139 279 279 279 F F F 2 040-059 140 140 140 282 282 282 F F F 3 060-079 126 126 126 257 257 257 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 139 139 139 279 279 279 F F F 6 120-139 138 138 138 278 278 278 F F F 7 140-159 131 131 131 266 266 266 F F F 8 160-179 140 140 140 281 281 281 F F F 9 180-199 141 141 141 285 285 285 F F F 10 200-219 138 138 138 276 276 276 F F F 11 220-239 140 140 140 281 281 281 F F F 12 240-259 138 138 138 278 278 278 F F F 13 260-279 138 138 138 277 277 277 F F F 14 280-299 138 138 138 277 277 277 F F F 15 300-319 136 136 136 273 273 273 F F F 16 320-339 138 138 138 278 278 278 F F F 17 340-359 140 140 140 283 283 283 F F F 18 360-379 140 140 140 283 283 283 F F F 19 380-399 136 136 136 274 274 274 F F F 20 400-419 140 140 140 280 280 280 F F F 21 420-439 141 141 141 285 285 285 F F F 22 440-459 139 139 139 279 279 279 F F F 23 460-479 141 141 141 285 285 285 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 140 140 140 284 284 284 F F F 27 540-559 140 140 140 284 284 284 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 139 139 139 279 279 279 F F F Max gen: 141, Max flush: 285 Begin loop 71 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.472363564Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.472718289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.472745669Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750} {"msg":"Crucible 1964b9c6-bdd3-4a65-8ab2-8c1e6248cbda has session id: ebd6eee4-b555-48a2-a13c-30794681cb59","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.473145804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Upstairs opts: Upstairs UUID: 1964b9c6-bdd3-4a65-8ab2-8c1e6248cbda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.473190875Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Crucible stats registered with UUID: 1964b9c6-bdd3-4a65-8ab2-8c1e6248cbda","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.473208275Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.473447908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750} {"msg":"1964b9c6-bdd3-4a65-8ab2-8c1e6248cbda active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.473565859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.47358372Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.47359853Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.47361221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.47362586Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.47363928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.4736548Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.473691101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.473708231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.473722321Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_connection connected from Ok(127.0.0.1:46426)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.473863893Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_connection connected from Ok(127.0.0.1:59479)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.474096316Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:58.474180327Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:58.474203387Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:54:58.474219568Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.474231848Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.474245068Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.474355229Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.474732984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.474770875Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.474800065Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.474826375Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.475349172Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.475463474Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.475800248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.475856519Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.475871119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.47602091Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.476041111Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:54:58.476054871Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.475057098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_connection connected from Ok(127.0.0.1:50556)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.47521818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.476353365Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.476379565Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477102284Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477659201Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477684072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[0]R flush_numbers[0..12]: [285, 279, 282, 257, 279, 279, 278, 266, 281, 285, 276, 281]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477703112Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[0]R generation[0..12]: [141, 139, 140, 126, 139, 139, 138, 131, 140, 141, 138, 140]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477720992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477739052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[1]R flush_numbers[0..12]: [285, 279, 282, 257, 279, 279, 278, 266, 281, 285, 276, 281]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477756623Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[1]R generation[0..12]: [141, 139, 140, 126, 139, 139, 138, 131, 140, 141, 138, 140]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477780733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477799343Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[2]R flush_numbers[0..12]: [285, 279, 282, 257, 279, 279, 278, 266, 281, 285, 276, 281]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477817013Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[2]R generation[0..12]: [141, 139, 140, 126, 139, 139, 138, 131, 140, 141, 138, 140]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477834953Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477856434Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477872914Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477889574Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477905905Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477922105Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"":"downstairs","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477938715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477953735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"0","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477970175Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"1","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.477986875Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"client":"2","":"downstairs","session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.478003716Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.478021646Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"1964b9c6-bdd3-4a65-8ab2-8c1e6248cbda is now active with session: ebd6eee4-b555-48a2-a13c-30794681cb59","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.478037776Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.478052906Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750,"session_id":"ebd6eee4-b555-48a2-a13c-30794681cb59"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:08.478068227Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 390 len 4096 data: 12 02/30 Read block 399 len 40960 03/30 Write block 147 len 28672 data: 15 13 13 10 10 8 7 04/30 Read block 535 len 28672 05/30 Write block 114 len 4096 data: 9 06/30 Read block 343 len 4096 07/30 Flush 08/30 Write block 16 len 4096 data: 11 09/30 Write block 104 len 28672 data: 13 11 11 11 11 10 9 10/30 Read block 344 len 24576 11/30 Write block 206 len 32768 data: 7 8 8 10 12 12 11 10 12/30 Write block 22 len 20480 data: 11 8 9 10 10 13/30 Write block 361 len 8192 data: 11 5 14/30 Flush 15/30 Write block 545 len 20480 data: 6 7 6 8 11 16/30 Flush 17/30 Write block 329 len 24576 data: 10 10 10 11 9 11 18/30 Flush 19/30 Write block 391 len 32768 data: 11 11 13 12 12 14 15 14 20/30 Write block 76 len 8192 data: 11 12 21/30 Write block 204 len 36864 data: 9 7 8 9 9 11 13 13 12 22/30 Read block 446 len 24576 23/30 Read block 583 len 24576 24/30 Write block 449 len 24576 data: 11 10 14 15 14 10 25/30 Read block 174 len 32768 26/30 Read block 337 len 40960 27/30 Write block 125 len 24576 data: 8 7 8 8 10 17 28/30 Write block 585 len 12288 data: 8 11 11 29/30 Read block 350 len 12288 30/30 Write block 153 len 8192 data: 8 9 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:55:08.506369896Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 287 287 287 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 140 140 140 282 282 282 F F F 3 060-079 126 126 126 257 257 257 T T T 4 080-099 139 139 139 279 279 279 F F F 5 100-119 142 142 142 287 287 287 F F F 6 120-139 138 138 138 278 278 278 T T T 7 140-159 142 142 142 286 286 286 T T T 8 160-179 140 140 140 281 281 281 F F F 9 180-199 141 141 141 285 285 285 F F F 10 200-219 142 142 142 287 287 287 T T T 11 220-239 140 140 140 281 281 281 F F F 12 240-259 138 138 138 278 278 278 F F F 13 260-279 138 138 138 277 277 277 F F F 14 280-299 138 138 138 277 277 277 F F F 15 300-319 136 136 136 273 273 273 F F F 16 320-339 142 142 142 289 289 289 F F F 17 340-359 140 140 140 283 283 283 F F F 18 360-379 142 142 142 287 287 287 F F F 19 380-399 142 142 142 286 286 286 T T T 20 400-419 140 140 140 280 280 280 F F F 21 420-439 141 141 141 285 285 285 F F F 22 440-459 139 139 139 279 279 279 T T T 23 460-479 141 141 141 285 285 285 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 140 140 140 284 284 284 F F F 27 540-559 142 142 142 288 288 288 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 139 139 139 279 279 279 T T T Max gen: 142, Max flush: 289 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 287 287 287 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 140 140 140 282 282 282 F F F 3 060-079 143 143 143 290 290 290 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 142 142 142 287 287 287 F F F 6 120-139 143 143 143 290 290 290 F F F 7 140-159 143 143 143 290 290 290 F F F 8 160-179 140 140 140 281 281 281 F F F 9 180-199 141 141 141 285 285 285 F F F 10 200-219 143 143 143 290 290 290 F F F 11 220-239 140 140 140 281 281 281 F F F 12 240-259 138 138 138 278 278 278 F F F 13 260-279 138 138 138 277 277 277 F F F 14 280-299 138 138 138 277 277 277 F F F 15 300-319 136 136 136 273 273 273 F F F 16 320-339 142 142 142 289 289 289 F F F 17 340-359 140 140 140 283 283 283 F F F 18 360-379 142 142 142 287 287 287 F F F 19 380-399 143 143 143 290 290 290 F F F 20 400-419 140 140 140 280 280 280 F F F 21 420-439 141 141 141 285 285 285 F F F 22 440-459 143 143 143 290 290 290 F F F 23 460-479 141 141 141 285 285 285 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 140 140 140 284 284 284 F F F 27 540-559 142 142 142 288 288 288 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 143 143 143 290 290 290 F F F Max gen: 143, Max flush: 290 Begin loop 72 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.761113708Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.761418912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.761447453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758} {"msg":"Crucible b9e5e1f4-a67a-4720-9efa-585138ce562d has session id: 697e722d-58e7-42fc-8f11-c36cf77bbc63","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.761797627Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Upstairs opts: Upstairs UUID: b9e5e1f4-a67a-4720-9efa-585138ce562d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.761844428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Crucible stats registered with UUID: b9e5e1f4-a67a-4720-9efa-585138ce562d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.761868998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762178222Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758} {"msg":"b9e5e1f4-a67a-4720-9efa-585138ce562d active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762203853Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762219053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762233833Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762248003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762261903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762275403Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762288854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762301994Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762315784Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762333644Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:55:22.762530517Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:55:22.762566987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_connection connected from Ok(127.0.0.1:35558)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762593457Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:55:22.762623298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_connection connected from Ok(127.0.0.1:42823)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762667688Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762696209Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762720089Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.762851491Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.763415298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.763453518Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.763482319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.763556379Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.764117017Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.764229808Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.764613413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.764640343Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.764655193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.764718884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.764737365Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:22.764750935Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.764193518Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_connection connected from Ok(127.0.0.1:56137)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.764430631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.765666216Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.765699397Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766239504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.76677982Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766807751Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[0]R flush_numbers[0..12]: [287, 287, 282, 290, 279, 287, 290, 290, 281, 285, 290, 281]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766824191Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[0]R generation[0..12]: [142, 142, 140, 143, 139, 142, 143, 143, 140, 141, 143, 140]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766839071Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766859772Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[1]R flush_numbers[0..12]: [287, 287, 282, 290, 279, 287, 290, 290, 281, 285, 290, 281]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766874692Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[1]R generation[0..12]: [142, 142, 140, 143, 139, 142, 143, 143, 140, 141, 143, 140]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766888652Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766902902Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[2]R flush_numbers[0..12]: [287, 287, 282, 290, 279, 287, 290, 290, 281, 285, 290, 281]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766916782Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[2]R generation[0..12]: [142, 142, 140, 143, 139, 142, 143, 143, 140, 141, 143, 140]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766930542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:55:32.766944662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766961133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766974483Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.766987603Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767000833Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"":"downstairs","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767014134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767026094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"0","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767039544Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"1","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767054984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"client":"2","":"downstairs","session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767068494Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767080844Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"b9e5e1f4-a67a-4720-9efa-585138ce562d is now active with session: 697e722d-58e7-42fc-8f11-c36cf77bbc63","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767094044Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767106115Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758,"session_id":"697e722d-58e7-42fc-8f11-c36cf77bbc63"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:32.767118475Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 390 len 32768 data: 13 12 12 14 13 13 15 16 02/30 Write block 404 len 36864 data: 10 9 9 9 9 8 7 5 6 03/30 Write block 371 len 32768 data: 14 11 13 15 12 14 18 17 04/30 Read block 213 len 12288 05/30 Write block 122 len 40960 data: 9 7 8 9 8 9 9 11 18 17 06/30 Read block 410 len 16384 07/30 Write block 191 len 12288 data: 12 14 12 08/30 Write block 254 len 28672 data: 15 17 15 12 9 11 13 09/30 Write block 121 len 36864 data: 9 10 8 9 10 9 10 10 12 10/30 Read block 41 len 32768 11/30 Write block 461 len 32768 data: 8 9 10 12 12 10 9 8 12/30 Write block 529 len 40960 data: 17 16 17 15 9 10 8 10 12 14 13/30 Write block 424 len 16384 data: 12 12 12 13 14/30 Read block 114 len 28672 15/30 Read block 454 len 20480 16/30 Write block 300 len 28672 data: 7 8 10 11 11 12 12 17/30 Write block 219 len 12288 data: 8 8 13 18/30 Write block 318 len 36864 data: 10 13 12 12 15 15 14 13 14 19/30 Read block 131 len 12288 20/30 Write block 403 len 12288 data: 12 11 10 21/30 Write block 348 len 36864 data: 14 10 10 10 11 12 11 12 13 22/30 Read block 33 len 36864 23/30 Read block 332 len 36864 24/30 Read block 476 len 40960 25/30 Read block 73 len 40960 26/30 Write block 470 len 28672 data: 9 8 7 10 10 14 17 27/30 Read block 148 len 40960 28/30 Read block 2 len 40960 29/30 Write block 288 len 32768 data: 9 10 9 9 11 12 11 8 30/30 Read block 170 len 12288 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 3 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 8 Done Done Done false 31 Acked 1030 Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:55:32.794427681Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 287 287 287 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 140 140 140 282 282 282 F F F 3 060-079 143 143 143 290 290 290 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 142 142 142 287 287 287 F F F 6 120-139 144 144 144 291 291 291 F F F 7 140-159 143 143 143 290 290 290 F F F 8 160-179 140 140 140 281 281 281 F F F 9 180-199 144 144 144 291 291 291 F F F 10 200-219 144 144 144 291 291 291 F F F 11 220-239 144 144 144 291 291 291 F F F 12 240-259 144 144 144 291 291 291 F F F 13 260-279 144 144 144 291 291 291 F F F 14 280-299 138 138 138 277 277 277 T T T 15 300-319 144 144 144 291 291 291 F F F 16 320-339 144 144 144 291 291 291 F F F 17 340-359 140 140 140 283 283 283 T T T 18 360-379 144 144 144 291 291 291 F F F 19 380-399 144 144 144 291 291 291 F F F 20 400-419 144 144 144 291 291 291 T T T 21 420-439 144 144 144 291 291 291 F F F 22 440-459 143 143 143 290 290 290 F F F 23 460-479 144 144 144 291 291 291 T T T 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 144 144 144 291 291 291 F F F 27 540-559 142 142 142 288 288 288 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 143 143 143 290 290 290 F F F Max gen: 144, Max flush: 291 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 287 287 287 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 140 140 140 282 282 282 F F F 3 060-079 143 143 143 290 290 290 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 142 142 142 287 287 287 F F F 6 120-139 144 144 144 291 291 291 F F F 7 140-159 143 143 143 290 290 290 F F F 8 160-179 140 140 140 281 281 281 F F F 9 180-199 144 144 144 291 291 291 F F F 10 200-219 144 144 144 291 291 291 F F F 11 220-239 144 144 144 291 291 291 F F F 12 240-259 144 144 144 291 291 291 F F F 13 260-279 144 144 144 291 291 291 F F F 14 280-299 145 145 145 292 292 292 F F F 15 300-319 144 144 144 291 291 291 F F F 16 320-339 144 144 144 291 291 291 F F F 17 340-359 145 145 145 292 292 292 F F F 18 360-379 144 144 144 291 291 291 F F F 19 380-399 144 144 144 291 291 291 F F F 20 400-419 145 145 145 292 292 292 F F F 21 420-439 144 144 144 291 291 291 F F F 22 440-459 143 143 143 290 290 290 F F F 23 460-479 145 145 145 292 292 292 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 144 144 144 291 291 291 F F F 27 540-559 142 142 142 288 288 288 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 143 143 143 290 290 290 F F F Max gen: 145, Max flush: 292 Begin loop 73 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.007540346Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008019421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008047482Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766} {"msg":"Crucible 0fee173b-8983-42c6-b849-a359603854c2 has session id: eb89091d-8f80-4767-83f0-a1134aa34f8b","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008287065Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Upstairs opts: Upstairs UUID: 0fee173b-8983-42c6-b849-a359603854c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008330445Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Crucible stats registered with UUID: 0fee173b-8983-42c6-b849-a359603854c2","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008354886Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.00868268Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766} {"msg":"0fee173b-8983-42c6-b849-a359603854c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008765331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008790121Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008805882Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008828392Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008854352Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008880243Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008918173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008949403Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008973364Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.008995114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65516)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.009114066Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47808)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.009151106Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:55:47.009426639Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:55:47.00946707Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:55:47.00949788Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.009522671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.009546281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.009580501Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.009982137Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.010020417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.010035887Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.010053807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.010630035Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.010652815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.011188872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.011217282Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.011241693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.011256903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.011270943Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:47.011284393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.010496643Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39687)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.010755826Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.011714708Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.011747969Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.012373577Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.012974834Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.012997505Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[0]R flush_numbers[0..12]: [287, 287, 282, 290, 279, 287, 291, 290, 281, 291, 291, 291]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013019075Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[0]R generation[0..12]: [142, 142, 140, 143, 139, 142, 144, 143, 140, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013034255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013048815Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[1]R flush_numbers[0..12]: [287, 287, 282, 290, 279, 287, 291, 290, 281, 291, 291, 291]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013062925Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[1]R generation[0..12]: [142, 142, 140, 143, 139, 142, 144, 143, 140, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013076906Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013091076Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[2]R flush_numbers[0..12]: [287, 287, 282, 290, 279, 287, 291, 290, 281, 291, 291, 291]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013105076Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[2]R generation[0..12]: [142, 142, 140, 143, 139, 142, 144, 143, 140, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013118926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013133036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013146197Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2024-07-10T15:55:57.013159537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013176667Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013189747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"":"downstairs","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013203047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013217698Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"0","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013231348Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"1","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013244698Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"client":"2","":"downstairs","session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013258218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013270818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"0fee173b-8983-42c6-b849-a359603854c2 is now active with session: eb89091d-8f80-4767-83f0-a1134aa34f8b","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013284008Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013296088Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766,"session_id":"eb89091d-8f80-4767-83f0-a1134aa34f8b"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:55:57.013308419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 293 len 20480 data: 13 12 9 9 9 02/30 Write block 61 len 16384 data: 12 8 8 10 03/30 Read block 332 len 40960 04/30 Write block 457 len 16384 data: 6 6 5 7 05/30 Write block 277 len 32768 data: 7 10 9 8 8 9 7 6 06/30 Flush 07/30 Write block 398 len 8192 data: 15 15 08/30 Read block 350 len 8192 09/30 Read block 509 len 28672 10/30 Read block 0 len 32768 11/30 Write block 522 len 20480 data: 6 7 9 10 16 12/30 Write block 377 len 20480 data: 19 18 17 15 12 13/30 Write block 464 len 28672 data: 13 13 11 10 9 8 10 14/30 Write block 530 len 24576 data: 17 18 16 10 11 9 15/30 Write block 11 len 4096 data: 10 16/30 Read block 329 len 32768 17/30 Flush 18/30 Write block 263 len 16384 data: 9 9 8 8 19/30 Flush 20/30 Write block 43 len 8192 data: 14 14 21/30 Read block 263 len 20480 22/30 Flush 23/30 Write block 176 len 16384 data: 9 10 9 9 24/30 Read block 12 len 12288 25/30 Read block 511 len 28672 26/30 Read block 44 len 8192 27/30 Write block 212 len 16384 data: 13 11 9 8 28/30 Write block 108 len 4096 data: 12 29/30 Write block 155 len 8192 data: 9 12 30/30 Read block 245 len 24576 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:55:57.040857338Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 294 294 294 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 146 146 146 296 296 296 F F F 3 060-079 146 146 146 293 293 293 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 142 142 142 287 287 287 T T T 6 120-139 144 144 144 291 291 291 F F F 7 140-159 143 143 143 290 290 290 T T T 8 160-179 140 140 140 281 281 281 T T T 9 180-199 144 144 144 291 291 291 F F F 10 200-219 144 144 144 291 291 291 T T T 11 220-239 144 144 144 291 291 291 F F F 12 240-259 144 144 144 291 291 291 F F F 13 260-279 146 146 146 295 295 295 F F F 14 280-299 146 146 146 293 293 293 F F F 15 300-319 144 144 144 291 291 291 F F F 16 320-339 144 144 144 291 291 291 F F F 17 340-359 145 145 145 292 292 292 F F F 18 360-379 146 146 146 294 294 294 F F F 19 380-399 146 146 146 294 294 294 F F F 20 400-419 145 145 145 292 292 292 F F F 21 420-439 144 144 144 291 291 291 F F F 22 440-459 146 146 146 293 293 293 F F F 23 460-479 146 146 146 294 294 294 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 146 146 146 294 294 294 F F F 27 540-559 142 142 142 288 288 288 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 143 143 143 290 290 290 F F F Max gen: 146, Max flush: 296 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 294 294 294 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 146 146 146 296 296 296 F F F 3 060-079 146 146 146 293 293 293 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 147 147 147 297 297 297 F F F 6 120-139 144 144 144 291 291 291 F F F 7 140-159 147 147 147 297 297 297 F F F 8 160-179 147 147 147 297 297 297 F F F 9 180-199 144 144 144 291 291 291 F F F 10 200-219 147 147 147 297 297 297 F F F 11 220-239 144 144 144 291 291 291 F F F 12 240-259 144 144 144 291 291 291 F F F 13 260-279 146 146 146 295 295 295 F F F 14 280-299 146 146 146 293 293 293 F F F 15 300-319 144 144 144 291 291 291 F F F 16 320-339 144 144 144 291 291 291 F F F 17 340-359 145 145 145 292 292 292 F F F 18 360-379 146 146 146 294 294 294 F F F 19 380-399 146 146 146 294 294 294 F F F 20 400-419 145 145 145 292 292 292 F F F 21 420-439 144 144 144 291 291 291 F F F 22 440-459 146 146 146 293 293 293 F F F 23 460-479 146 146 146 294 294 294 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 146 146 146 294 294 294 F F F 27 540-559 142 142 142 288 288 288 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 143 143 143 290 290 290 F F F Max gen: 147, Max flush: 297 Begin loop 74 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.252322162Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253096312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253127292Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774} {"msg":"Crucible f99f759f-149b-4804-b0ab-2191f076a84a has session id: b25a34a7-4a96-4645-9567-7efd249f4842","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253151092Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Upstairs opts: Upstairs UUID: f99f759f-149b-4804-b0ab-2191f076a84a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253294774Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Crucible stats registered with UUID: f99f759f-149b-4804-b0ab-2191f076a84a","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253329805Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253634489Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774} {"msg":"f99f759f-149b-4804-b0ab-2191f076a84a active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.25372161Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.25374005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.25375466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.25376835Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.25378181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253797731Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253827401Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253845111Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253860101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.253873792Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:11.254213016Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:11.254248096Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:11.254276857Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.254301907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.254328727Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.254353818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_connection connected from Ok(127.0.0.1:47839)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.254412288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_connection connected from Ok(127.0.0.1:34978)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.254439139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.255114347Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.255140848Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.255155348Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.255173148Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.255787516Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.255811956Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.256315492Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.256340083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.256354163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.256411094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.256429084Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:11.256442644Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.255656254Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_connection connected from Ok(127.0.0.1:62305)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.255751255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.256869229Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.25690345Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.257506007Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258073765Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258099305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[0]R flush_numbers[0..12]: [294, 287, 296, 293, 279, 297, 291, 297, 297, 291, 297, 291]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258118265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[0]R generation[0..12]: [146, 142, 146, 146, 139, 147, 144, 147, 147, 144, 147, 144]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258136055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258153896Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[1]R flush_numbers[0..12]: [294, 287, 296, 293, 279, 297, 291, 297, 297, 291, 297, 291]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258171746Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[1]R generation[0..12]: [146, 142, 146, 146, 139, 147, 144, 147, 147, 144, 147, 144]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258188896Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258206706Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[2]R flush_numbers[0..12]: [294, 287, 296, 293, 279, 297, 291, 297, 297, 291, 297, 291]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258224287Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[2]R generation[0..12]: [146, 142, 146, 146, 139, 147, 144, 147, 147, 144, 147, 144]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258241777Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258262667Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258278997Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258295797Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258311908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258328168Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"":"downstairs","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258347458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258362738Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"0","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258379549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"1","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258396179Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"client":"2","":"downstairs","session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258412669Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258428019Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"f99f759f-149b-4804-b0ab-2191f076a84a is now active with session: b25a34a7-4a96-4645-9567-7efd249f4842","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258444249Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.258459239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774,"session_id":"b25a34a7-4a96-4645-9567-7efd249f4842"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:21.25847456Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 263 len 20480 data: 10 10 9 9 6 02/30 Write block 220 len 12288 data: 9 14 14 03/30 Write block 202 len 36864 data: 8 9 10 8 9 10 10 12 14 04/30 Read block 566 len 16384 05/30 Flush 06/30 Write block 268 len 28672 data: 3 4 6 4 7 9 9 07/30 Read block 241 len 8192 08/30 Write block 388 len 12288 data: 9 11 14 09/30 Write block 387 len 16384 data: 9 10 12 15 10/30 Read block 468 len 28672 11/30 Write block 330 len 40960 data: 11 11 12 10 12 9 9 9 9 12 12/30 Read block 456 len 8192 13/30 Write block 245 len 32768 data: 14 15 14 15 14 14 15 13 14/30 Write block 75 len 8192 data: 11 12 15/30 Read block 236 len 20480 16/30 Write block 550 len 28672 data: 13 13 13 11 10 9 6 17/30 Write block 143 len 32768 data: 14 14 12 13 16 14 14 11 18/30 Write block 382 len 8192 data: 12 7 19/30 Read block 576 len 12288 20/30 Write block 445 len 20480 data: 11 11 14 12 12 21/30 Read block 9 len 40960 22/30 Write block 207 len 12288 data: 11 11 13 23/30 Write block 461 len 40960 data: 9 10 11 14 14 12 11 10 9 11 24/30 Read block 196 len 12288 25/30 Read block 593 len 28672 26/30 Write block 218 len 32768 data: 7 9 10 15 15 13 15 13 27/30 Write block 369 len 40960 data: 12 15 15 12 14 16 13 15 20 19 28/30 Read block 269 len 8192 29/30 Read block 303 len 20480 30/30 Write block 401 len 12288 data: 13 12 13 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:21.284111545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 294 294 294 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 146 146 146 296 296 296 F F F 3 060-079 148 148 148 299 299 299 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 147 147 147 297 297 297 F F F 6 120-139 144 144 144 291 291 291 F F F 7 140-159 148 148 148 299 299 299 F F F 8 160-179 147 147 147 297 297 297 F F F 9 180-199 144 144 144 291 291 291 F F F 10 200-219 148 148 148 298 298 298 T T T 11 220-239 148 148 148 298 298 298 T T T 12 240-259 148 148 148 299 299 299 F F F 13 260-279 148 148 148 299 299 299 F F F 14 280-299 146 146 146 293 293 293 F F F 15 300-319 144 144 144 291 291 291 F F F 16 320-339 148 148 148 299 299 299 F F F 17 340-359 145 145 145 292 292 292 F F F 18 360-379 146 146 146 294 294 294 T T T 19 380-399 148 148 148 299 299 299 F F F 20 400-419 145 145 145 292 292 292 T T T 21 420-439 144 144 144 291 291 291 F F F 22 440-459 146 146 146 293 293 293 T T T 23 460-479 146 146 146 294 294 294 T T T 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 146 146 146 294 294 294 F F F 27 540-559 148 148 148 299 299 299 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 143 143 143 290 290 290 F F F Max gen: 148, Max flush: 299 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 294 294 294 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 146 146 146 296 296 296 F F F 3 060-079 148 148 148 299 299 299 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 147 147 147 297 297 297 F F F 6 120-139 144 144 144 291 291 291 F F F 7 140-159 148 148 148 299 299 299 F F F 8 160-179 147 147 147 297 297 297 F F F 9 180-199 144 144 144 291 291 291 F F F 10 200-219 149 149 149 300 300 300 F F F 11 220-239 149 149 149 300 300 300 F F F 12 240-259 148 148 148 299 299 299 F F F 13 260-279 148 148 148 299 299 299 F F F 14 280-299 146 146 146 293 293 293 F F F 15 300-319 144 144 144 291 291 291 F F F 16 320-339 148 148 148 299 299 299 F F F 17 340-359 145 145 145 292 292 292 F F F 18 360-379 149 149 149 300 300 300 F F F 19 380-399 148 148 148 299 299 299 F F F 20 400-419 149 149 149 300 300 300 F F F 21 420-439 144 144 144 291 291 291 F F F 22 440-459 149 149 149 300 300 300 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 136 136 136 273 273 273 F F F 26 520-539 146 146 146 294 294 294 F F F 27 540-559 148 148 148 299 299 299 F F F 28 560-579 139 139 139 279 279 279 F F F 29 580-599 143 143 143 290 290 290 F F F Max gen: 149, Max flush: 300 Begin loop 75 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.523019197Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.523365181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.523392142Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782} {"msg":"Crucible 1d3b74c1-45bf-4b29-bc54-91a4040bd32e has session id: df13f529-b50a-4231-9207-74472d7ee905","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.523509133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Upstairs opts: Upstairs UUID: 1d3b74c1-45bf-4b29-bc54-91a4040bd32e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.523668985Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Crucible stats registered with UUID: 1d3b74c1-45bf-4b29-bc54-91a4040bd32e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.523702866Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524098981Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782} {"msg":"1d3b74c1-45bf-4b29-bc54-91a4040bd32e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524210562Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524230183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524245373Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524259213Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524287573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524313613Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524340014Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524364474Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524389005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524413125Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:35.524516936Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:35.524553707Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:35.524583157Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524605247Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524626657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_connection connected from Ok(127.0.0.1:61164)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524648948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_connection connected from Ok(127.0.0.1:40716)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524885071Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.524939512Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.525444818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.525471798Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.525488138Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.525511609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.526099546Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.526126787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.526643533Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.526678034Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.526700294Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.526719364Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.526740304Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:35.526760904Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.52559438Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_connection connected from Ok(127.0.0.1:45824)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.525748382Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.526678903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.526706214Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.527361092Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.527928679Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.52795309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[0]R flush_numbers[0..12]: [294, 287, 296, 299, 279, 297, 291, 299, 297, 291, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.52796889Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[0]R generation[0..12]: [146, 142, 146, 148, 139, 147, 144, 148, 147, 144, 149, 149]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.52798344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.52799794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[1]R flush_numbers[0..12]: [294, 287, 296, 299, 279, 297, 291, 299, 297, 291, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528012031Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[1]R generation[0..12]: [146, 142, 146, 148, 139, 147, 144, 148, 147, 144, 149, 149]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528026071Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528040381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[2]R flush_numbers[0..12]: [294, 287, 296, 299, 279, 297, 291, 299, 297, 291, 300, 300]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528054261Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[2]R generation[0..12]: [146, 142, 146, 148, 139, 147, 144, 148, 147, 144, 149, 149]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528068181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528082401Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528095541Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528117522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528130992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528144152Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"":"downstairs","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528157532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528169542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"0","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528182973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"1","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528196203Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"client":"2","":"downstairs","session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528209683Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528221933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"1d3b74c1-45bf-4b29-bc54-91a4040bd32e is now active with session: df13f529-b50a-4231-9207-74472d7ee905","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528235033Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528247274Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782,"session_id":"df13f529-b50a-4231-9207-74472d7ee905"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:45.528259523Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 386 len 24576 data: 13 10 11 13 16 13 02/30 Write block 388 len 40960 data: 12 14 17 14 13 15 14 14 16 17 03/30 Flush 04/30 Write block 358 len 4096 data: 12 05/30 Write block 521 len 4096 data: 6 06/30 Read block 216 len 24576 07/30 Read block 527 len 8192 08/30 Flush 09/30 Write block 114 len 4096 data: 10 10/30 Read block 52 len 40960 11/30 Write block 328 len 8192 data: 11 11 12/30 Write block 432 len 4096 data: 13 13/30 Write block 434 len 16384 data: 7 8 8 9 14/30 Write block 445 len 16384 data: 12 12 15 13 15/30 Read block 23 len 20480 16/30 Read block 138 len 8192 17/30 Read block 175 len 4096 18/30 Read block 266 len 12288 19/30 Read block 509 len 20480 20/30 Read block 240 len 36864 21/30 Read block 526 len 24576 22/30 Write block 234 len 12288 data: 14 15 16 23/30 Write block 189 len 24576 data: 10 9 13 15 13 12 24/30 Write block 496 len 28672 data: 11 12 13 13 13 13 12 25/30 Flush 26/30 Write block 257 len 12288 data: 13 10 12 27/30 Write block 572 len 36864 data: 9 8 11 11 14 13 15 12 11 28/30 Read block 66 len 4096 29/30 Read block 403 len 24576 30/30 Read block 234 len 32768 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:45.554252664Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 294 294 294 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 146 146 146 296 296 296 F F F 3 060-079 148 148 148 299 299 299 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 150 150 150 303 303 303 F F F 6 120-139 144 144 144 291 291 291 F F F 7 140-159 148 148 148 299 299 299 F F F 8 160-179 147 147 147 297 297 297 F F F 9 180-199 150 150 150 304 304 304 F F F 10 200-219 149 149 149 300 300 300 F F F 11 220-239 150 150 150 304 304 304 F F F 12 240-259 148 148 148 299 299 299 T T T 13 260-279 148 148 148 299 299 299 F F F 14 280-299 146 146 146 293 293 293 F F F 15 300-319 144 144 144 291 291 291 F F F 16 320-339 150 150 150 303 303 303 F F F 17 340-359 150 150 150 302 302 302 F F F 18 360-379 149 149 149 300 300 300 F F F 19 380-399 150 150 150 301 301 301 F F F 20 400-419 149 149 149 300 300 300 F F F 21 420-439 150 150 150 303 303 303 F F F 22 440-459 150 150 150 303 303 303 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 150 150 150 304 304 304 F F F 25 500-519 150 150 150 304 304 304 F F F 26 520-539 150 150 150 302 302 302 F F F 27 540-559 148 148 148 299 299 299 F F F 28 560-579 139 139 139 279 279 279 T T T 29 580-599 143 143 143 290 290 290 T T T Max gen: 150, Max flush: 304 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 294 294 294 F F F 1 020-039 142 142 142 287 287 287 F F F 2 040-059 146 146 146 296 296 296 F F F 3 060-079 148 148 148 299 299 299 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 150 150 150 303 303 303 F F F 6 120-139 144 144 144 291 291 291 F F F 7 140-159 148 148 148 299 299 299 F F F 8 160-179 147 147 147 297 297 297 F F F 9 180-199 150 150 150 304 304 304 F F F 10 200-219 149 149 149 300 300 300 F F F 11 220-239 150 150 150 304 304 304 F F F 12 240-259 151 151 151 305 305 305 F F F 13 260-279 148 148 148 299 299 299 F F F 14 280-299 146 146 146 293 293 293 F F F 15 300-319 144 144 144 291 291 291 F F F 16 320-339 150 150 150 303 303 303 F F F 17 340-359 150 150 150 302 302 302 F F F 18 360-379 149 149 149 300 300 300 F F F 19 380-399 150 150 150 301 301 301 F F F 20 400-419 149 149 149 300 300 300 F F F 21 420-439 150 150 150 303 303 303 F F F 22 440-459 150 150 150 303 303 303 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 150 150 150 304 304 304 F F F 25 500-519 150 150 150 304 304 304 F F F 26 520-539 150 150 150 302 302 302 F F F 27 540-559 148 148 148 299 299 299 F F F 28 560-579 151 151 151 305 305 305 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 151, Max flush: 305 Begin loop 76 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.757799776Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.758135541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.758159811Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790} {"msg":"Crucible 57a3525c-e20a-47d1-b90a-b0dab223b200 has session id: a06b1088-83a2-4ff1-a894-13d42e02779d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.758464285Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Upstairs opts: Upstairs UUID: 57a3525c-e20a-47d1-b90a-b0dab223b200, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.758511236Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Crucible stats registered with UUID: 57a3525c-e20a-47d1-b90a-b0dab223b200","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.758535006Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.758927701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790} {"msg":"57a3525c-e20a-47d1-b90a-b0dab223b200 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.758949511Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.758971231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.758996232Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759020272Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759046382Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759069803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759096183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759118093Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759141684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759166414Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:59.759373776Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:59.759410907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36910)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759435107Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:56:59.759462578Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759485288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759509648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45210)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759534209Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.759567589Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.760082965Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.760111086Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.760126836Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.760140816Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.760794525Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.760831685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.761310021Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.761333961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.761348632Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.761423162Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.761441383Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:56:59.761454693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.760583762Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51239)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.760758694Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.76197102Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.76202172Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.762580077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763137114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763164145Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[0]R flush_numbers[0..12]: [294, 287, 296, 299, 279, 303, 291, 299, 297, 304, 300, 304]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763180445Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[0]R generation[0..12]: [146, 142, 146, 148, 139, 150, 144, 148, 147, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763195385Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763209995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[1]R flush_numbers[0..12]: [294, 287, 296, 299, 279, 303, 291, 299, 297, 304, 300, 304]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763225155Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[1]R generation[0..12]: [146, 142, 146, 148, 139, 150, 144, 148, 147, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763239265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763253676Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[2]R flush_numbers[0..12]: [294, 287, 296, 299, 279, 303, 291, 299, 297, 304, 300, 304]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763267646Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[2]R generation[0..12]: [146, 142, 146, 148, 139, 150, 144, 148, 147, 150, 149, 150]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763281406Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763295666Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763312406Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763331217Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763344947Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763358277Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"":"downstairs","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763371767Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763383978Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"0","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763397298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"1","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763410588Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"client":"2","":"downstairs","session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763424098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763436458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"57a3525c-e20a-47d1-b90a-b0dab223b200 is now active with session: a06b1088-83a2-4ff1-a894-13d42e02779d","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763449418Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763461568Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790,"session_id":"a06b1088-83a2-4ff1-a894-13d42e02779d"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:09.763474008Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 354 len 40960 data: 12 13 14 11 13 11 10 12 6 7 02/30 Flush 03/30 Write block 542 len 40960 data: 9 9 11 7 8 7 9 12 14 14 04/30 Read block 78 len 4096 05/30 Read block 85 len 4096 06/30 Read block 306 len 24576 07/30 Flush 08/30 Write block 18 len 16384 data: 14 13 13 12 09/30 Read block 289 len 8192 10/30 Write block 189 len 12288 data: 11 10 14 11/30 Write block 59 len 32768 data: 12 11 13 9 9 11 14 12 12/30 Write block 130 len 32768 data: 19 18 15 14 16 16 14 17 13/30 Read block 533 len 4096 14/30 Write block 548 len 16384 data: 10 13 15 15 15/30 Write block 253 len 20480 data: 12 16 18 16 14 16/30 Read block 299 len 24576 17/30 Read block 298 len 32768 18/30 Flush 19/30 Write block 305 len 40960 data: 13 13 12 10 9 9 5 5 6 9 20/30 Flush 21/30 Write block 271 len 8192 data: 5 8 22/30 Flush 23/30 Write block 67 len 36864 data: 9 10 11 13 12 11 14 11 12 24/30 Write block 216 len 4096 data: 8 25/30 Write block 374 len 20480 data: 17 14 16 21 20 26/30 Read block 108 len 16384 27/30 Write block 398 len 12288 data: 16 16 16 28/30 Write block 203 len 12288 data: 10 11 9 29/30 Write block 486 len 36864 data: 9 11 11 10 11 13 13 12 12 30/30 Write block 486 len 40960 data: 10 12 12 11 12 14 14 13 13 10 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Write 9 Done Done Done false 31 Acked 1030 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:57:09.793164035Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 308 308 308 F F F 1 020-039 152 152 152 308 308 308 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 152 152 152 308 308 308 T T T 4 080-099 139 139 139 279 279 279 F F F 5 100-119 150 150 150 303 303 303 F F F 6 120-139 152 152 152 308 308 308 F F F 7 140-159 148 148 148 299 299 299 F F F 8 160-179 147 147 147 297 297 297 F F F 9 180-199 152 152 152 308 308 308 F F F 10 200-219 149 149 149 300 300 300 T T T 11 220-239 150 150 150 304 304 304 F F F 12 240-259 152 152 152 308 308 308 F F F 13 260-279 152 152 152 311 311 311 F F F 14 280-299 146 146 146 293 293 293 F F F 15 300-319 152 152 152 310 310 310 F F F 16 320-339 150 150 150 303 303 303 F F F 17 340-359 152 152 152 306 306 306 F F F 18 360-379 152 152 152 306 306 306 T T T 19 380-399 150 150 150 301 301 301 T T T 20 400-419 149 149 149 300 300 300 T T T 21 420-439 150 150 150 303 303 303 F F F 22 440-459 150 150 150 303 303 303 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 150 150 150 304 304 304 T T T 25 500-519 150 150 150 304 304 304 F F F 26 520-539 150 150 150 302 302 302 F F F 27 540-559 152 152 152 308 308 308 F F F 28 560-579 151 151 151 305 305 305 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 152, Max flush: 311 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 308 308 308 F F F 1 020-039 152 152 152 308 308 308 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 153 153 153 312 312 312 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 150 150 150 303 303 303 F F F 6 120-139 152 152 152 308 308 308 F F F 7 140-159 148 148 148 299 299 299 F F F 8 160-179 147 147 147 297 297 297 F F F 9 180-199 152 152 152 308 308 308 F F F 10 200-219 153 153 153 312 312 312 F F F 11 220-239 150 150 150 304 304 304 F F F 12 240-259 152 152 152 308 308 308 F F F 13 260-279 152 152 152 311 311 311 F F F 14 280-299 146 146 146 293 293 293 F F F 15 300-319 152 152 152 310 310 310 F F F 16 320-339 150 150 150 303 303 303 F F F 17 340-359 152 152 152 306 306 306 F F F 18 360-379 153 153 153 312 312 312 F F F 19 380-399 153 153 153 312 312 312 F F F 20 400-419 153 153 153 312 312 312 F F F 21 420-439 150 150 150 303 303 303 F F F 22 440-459 150 150 150 303 303 303 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 153 153 153 312 312 312 F F F 25 500-519 150 150 150 304 304 304 F F F 26 520-539 150 150 150 302 302 302 F F F 27 540-559 152 152 152 308 308 308 F F F 28 560-579 151 151 151 305 305 305 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 153, Max flush: 312 Begin loop 77 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.035783624Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.03629701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.036327331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798} {"msg":"Crucible a7e4b9ac-0f89-4ead-a1bd-eab2bca5995a has session id: 862dddc4-0b74-4424-a330-670e1a2dcd6f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.036559734Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Upstairs opts: Upstairs UUID: a7e4b9ac-0f89-4ead-a1bd-eab2bca5995a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.036613594Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Crucible stats registered with UUID: a7e4b9ac-0f89-4ead-a1bd-eab2bca5995a","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.036638285Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.036932269Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798} {"msg":"a7e4b9ac-0f89-4ead-a1bd-eab2bca5995a active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.03706077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.03707983Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037094551Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037112391Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037126661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037140161Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037153901Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037222902Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037254293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037321053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35456)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037361754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:57:24.037545636Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:57:24.037584417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:57:24.037610477Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037632407Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51562)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037656068Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037680848Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.037747319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.038284686Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.038311676Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.038327266Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.038349036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.038928164Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.039091766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.03944Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.03946183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.039475901Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.039546142Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.039568392Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:24.039585312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.039232598Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46003)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.039362399Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.040534174Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.040608135Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041146552Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041700399Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041726269Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[0]R flush_numbers[0..12]: [308, 308, 308, 312, 279, 303, 308, 299, 297, 308, 312, 304]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041742319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[0]R generation[0..12]: [152, 152, 152, 153, 139, 150, 152, 148, 147, 152, 153, 150]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.04175749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.04177229Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[1]R flush_numbers[0..12]: [308, 308, 308, 312, 279, 303, 308, 299, 297, 308, 312, 304]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.04178662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[1]R generation[0..12]: [152, 152, 152, 153, 139, 150, 152, 148, 147, 152, 153, 150]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.04180081Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.04181504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[2]R flush_numbers[0..12]: [308, 308, 308, 312, 279, 303, 308, 299, 297, 308, 312, 304]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.04182915Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[2]R generation[0..12]: [152, 152, 152, 153, 139, 150, 152, 148, 147, 152, 153, 150]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.04184294Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041857231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041877861Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041891481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041904381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041917501Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"":"downstairs","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041930752Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041942722Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"0","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041956052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"1","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041969252Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"client":"2","":"downstairs","session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041982942Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.041995333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"a7e4b9ac-0f89-4ead-a1bd-eab2bca5995a is now active with session: 862dddc4-0b74-4424-a330-670e1a2dcd6f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.042008293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.042020353Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798,"session_id":"862dddc4-0b74-4424-a330-670e1a2dcd6f"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:34.042033053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 249 len 32768 data: 15 15 16 14 13 17 19 17 02/30 Read block 23 len 16384 03/30 Read block 410 len 16384 04/30 Write block 161 len 28672 data: 9 8 7 8 8 10 11 05/30 Read block 415 len 8192 06/30 Write block 427 len 32768 data: 14 14 15 13 13 14 11 8 07/30 Read block 406 len 32768 08/30 Write block 444 len 40960 data: 12 13 13 16 14 13 11 15 16 15 09/30 Flush 10/30 Write block 388 len 12288 data: 13 15 18 11/30 Read block 11 len 32768 12/30 Read block 478 len 4096 13/30 Read block 132 len 4096 14/30 Flush 15/30 Write block 415 len 32768 data: 4 5 5 9 7 7 7 9 16/30 Read block 298 len 28672 17/30 Read block 485 len 36864 18/30 Write block 391 len 12288 data: 15 14 16 19/30 Write block 537 len 36864 data: 13 15 12 11 8 10 10 12 8 20/30 Read block 461 len 24576 21/30 Flush 22/30 Write block 318 len 8192 data: 11 14 23/30 Flush 24/30 Write block 397 len 12288 data: 18 17 17 25/30 Write block 4 len 20480 data: 13 15 14 14 14 26/30 Read block 359 len 16384 27/30 Read block 354 len 12288 28/30 Read block 515 len 24576 29/30 Write block 102 len 8192 data: 10 15 30/30 Write block 264 len 16384 data: 11 10 10 7 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 2 Done Done Done false 31 Acked 1030 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:57:34.072408728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 308 308 308 T T T 1 020-039 152 152 152 308 308 308 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 153 153 153 312 312 312 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 150 150 150 303 303 303 T T T 6 120-139 152 152 152 308 308 308 F F F 7 140-159 148 148 148 299 299 299 F F F 8 160-179 154 154 154 313 313 313 F F F 9 180-199 152 152 152 308 308 308 F F F 10 200-219 153 153 153 312 312 312 F F F 11 220-239 150 150 150 304 304 304 F F F 12 240-259 154 154 154 313 313 313 F F F 13 260-279 152 152 152 311 311 311 T T T 14 280-299 146 146 146 293 293 293 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 150 150 150 303 303 303 F F F 17 340-359 152 152 152 306 306 306 F F F 18 360-379 153 153 153 312 312 312 F F F 19 380-399 154 154 154 315 315 315 T T T 20 400-419 154 154 154 315 315 315 F F F 21 420-439 154 154 154 315 315 315 F F F 22 440-459 154 154 154 313 313 313 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 153 153 153 312 312 312 F F F 25 500-519 150 150 150 304 304 304 F F F 26 520-539 154 154 154 316 316 316 F F F 27 540-559 154 154 154 316 316 316 F F F 28 560-579 151 151 151 305 305 305 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 154, Max flush: 317 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 318 318 318 F F F 1 020-039 152 152 152 308 308 308 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 153 153 153 312 312 312 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 155 155 155 318 318 318 F F F 6 120-139 152 152 152 308 308 308 F F F 7 140-159 148 148 148 299 299 299 F F F 8 160-179 154 154 154 313 313 313 F F F 9 180-199 152 152 152 308 308 308 F F F 10 200-219 153 153 153 312 312 312 F F F 11 220-239 150 150 150 304 304 304 F F F 12 240-259 154 154 154 313 313 313 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 146 146 146 293 293 293 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 150 150 150 303 303 303 F F F 17 340-359 152 152 152 306 306 306 F F F 18 360-379 153 153 153 312 312 312 F F F 19 380-399 155 155 155 318 318 318 F F F 20 400-419 154 154 154 315 315 315 F F F 21 420-439 154 154 154 315 315 315 F F F 22 440-459 154 154 154 313 313 313 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 153 153 153 312 312 312 F F F 25 500-519 150 150 150 304 304 304 F F F 26 520-539 154 154 154 316 316 316 F F F 27 540-559 154 154 154 316 316 316 F F F 28 560-579 151 151 151 305 305 305 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 155, Max flush: 318 Begin loop 78 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.286913101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.287199825Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.287226695Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806} {"msg":"Crucible 5adcb877-9e5d-499d-b7fe-0b855f6e665f has session id: e3096d4e-65b8-4c8d-a7fd-958115613c63","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.28764907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Upstairs opts: Upstairs UUID: 5adcb877-9e5d-499d-b7fe-0b855f6e665f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.287732291Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Crucible stats registered with UUID: 5adcb877-9e5d-499d-b7fe-0b855f6e665f","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.287752952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288089436Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806} {"msg":"5adcb877-9e5d-499d-b7fe-0b855f6e665f active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288162937Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288180207Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288198307Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288212847Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288236698Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288261118Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288285618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288310729Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288334609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288352709Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_connection connected from Ok(127.0.0.1:57301)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288679713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_connection connected from Ok(127.0.0.1:35743)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288715474Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:57:48.288820345Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:57:48.288852355Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:57:48.288919746Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288938167Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.288952577Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.289064688Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.289515754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.289542474Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.289557884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.289571455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.290188572Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.290225853Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.290722539Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.29075146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.29077721Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.29079206Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.29080629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:48.290819771Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.290242693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_connection connected from Ok(127.0.0.1:59807)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.290419145Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.291506549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.29154094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292156917Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292743145Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292766695Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[0]R flush_numbers[0..12]: [318, 308, 308, 312, 279, 318, 308, 299, 313, 308, 312, 304]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292782455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[0]R generation[0..12]: [155, 152, 152, 153, 139, 155, 152, 148, 154, 152, 153, 150]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292796835Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292811236Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[1]R flush_numbers[0..12]: [318, 308, 308, 312, 279, 318, 308, 299, 313, 308, 312, 304]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292825656Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[1]R generation[0..12]: [155, 152, 152, 153, 139, 155, 152, 148, 154, 152, 153, 150]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292839656Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292853946Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[2]R flush_numbers[0..12]: [318, 308, 308, 312, 279, 318, 308, 299, 313, 308, 312, 304]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292867796Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[2]R generation[0..12]: [155, 152, 152, 153, 139, 155, 152, 148, 154, 152, 153, 150]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292881666Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292895797Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292913157Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292930697Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292944097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292961978Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"":"downstairs","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.292985708Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.293007958Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"0","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.293031568Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"1","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.293055529Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"client":"2","":"downstairs","session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.293078699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.293099699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"5adcb877-9e5d-499d-b7fe-0b855f6e665f is now active with session: e3096d4e-65b8-4c8d-a7fd-958115613c63","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.29312332Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.29314469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806,"session_id":"e3096d4e-65b8-4c8d-a7fd-958115613c63"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:57:58.29316585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 71 len 28672 data: 13 12 15 12 13 13 13 02/30 Read block 268 len 40960 03/30 Write block 214 len 36864 data: 10 9 9 7 8 10 11 16 16 04/30 Read block 340 len 32768 05/30 Flush 06/30 Write block 283 len 20480 data: 8 7 7 7 9 07/30 Read block 60 len 28672 08/30 Read block 163 len 28672 09/30 Read block 415 len 40960 10/30 Read block 303 len 32768 11/30 Write block 419 len 8192 data: 8 8 12/30 Write block 524 len 36864 data: 10 11 17 16 17 18 18 19 17 13/30 Write block 155 len 20480 data: 10 13 11 14 12 14/30 Read block 361 len 32768 15/30 Read block 526 len 24576 16/30 Read block 236 len 8192 17/30 Read block 326 len 4096 18/30 Read block 343 len 12288 19/30 Write block 282 len 8192 data: 10 9 20/30 Flush 21/30 Write block 23 len 8192 data: 9 10 22/30 Read block 237 len 32768 23/30 Read block 205 len 4096 24/30 Read block 390 len 16384 25/30 Read block 193 len 4096 26/30 Flush 27/30 Write block 16 len 8192 data: 12 11 28/30 Write block 206 len 16384 data: 10 12 12 14 29/30 Read block 221 len 20480 30/30 Write block 332 len 4096 data: 13 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Read 5 Done Done Sent false 31 Acked 1030 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 2 4 Done 3 3 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:57:58.320510367Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 318 318 318 T T T 1 020-039 156 156 156 322 322 322 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 156 156 156 319 319 319 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 155 155 155 318 318 318 F F F 6 120-139 152 152 152 308 308 308 F F F 7 140-159 156 156 156 320 320 320 F F F 8 160-179 154 154 154 313 313 313 F F F 9 180-199 152 152 152 308 308 308 F F F 10 200-219 156 156 156 319 319 319 T T T 11 220-239 156 156 156 319 319 319 F F F 12 240-259 154 154 154 313 313 313 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 150 150 150 303 303 303 T T T 17 340-359 152 152 152 306 306 306 F F F 18 360-379 153 153 153 312 312 312 F F F 19 380-399 155 155 155 318 318 318 F F F 20 400-419 156 156 156 320 320 320 F F F 21 420-439 156 156 156 320 320 320 F F F 22 440-459 154 154 154 313 313 313 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 153 153 153 312 312 312 F F F 25 500-519 150 150 150 304 304 304 F F F 26 520-539 156 156 156 320 320 320 F F F 27 540-559 154 154 154 316 316 316 F F F 28 560-579 151 151 151 305 305 305 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 156, Max flush: 322 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 323 323 323 F F F 1 020-039 156 156 156 322 322 322 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 156 156 156 319 319 319 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 155 155 155 318 318 318 F F F 6 120-139 152 152 152 308 308 308 F F F 7 140-159 156 156 156 320 320 320 F F F 8 160-179 154 154 154 313 313 313 F F F 9 180-199 152 152 152 308 308 308 F F F 10 200-219 157 157 157 323 323 323 F F F 11 220-239 156 156 156 319 319 319 F F F 12 240-259 154 154 154 313 313 313 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 157 157 157 323 323 323 F F F 17 340-359 152 152 152 306 306 306 F F F 18 360-379 153 153 153 312 312 312 F F F 19 380-399 155 155 155 318 318 318 F F F 20 400-419 156 156 156 320 320 320 F F F 21 420-439 156 156 156 320 320 320 F F F 22 440-459 154 154 154 313 313 313 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 153 153 153 312 312 312 F F F 25 500-519 150 150 150 304 304 304 F F F 26 520-539 156 156 156 320 320 320 F F F 27 540-559 154 154 154 316 316 316 F F F 28 560-579 151 151 151 305 305 305 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 157, Max flush: 323 Begin loop 79 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.525225135Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.52561752Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.525648471Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814} {"msg":"Crucible 81f1bf93-91ce-4e12-9324-14975a83cca5 has session id: 329fbac0-f4f2-493c-81a6-4bb969df25f9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.525930554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Upstairs opts: Upstairs UUID: 81f1bf93-91ce-4e12-9324-14975a83cca5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.525975195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Crucible stats registered with UUID: 81f1bf93-91ce-4e12-9324-14975a83cca5","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.525999175Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526297159Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814} {"msg":"81f1bf93-91ce-4e12-9324-14975a83cca5 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.52641643Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.52644171Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526458021Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526476301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526490311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526503821Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526517491Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526547982Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526565812Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526579792Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38346)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.526774825Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:58:12.526967907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:58:12.527010208Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:58:12.527042748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.527065598Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.527091219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56037)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.527115099Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.527228931Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.527570885Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.527597335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.527612645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.527626705Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.528188423Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.528303024Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.528665879Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.528686009Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.528700109Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.528815641Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.528833721Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:12.528847211Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.528446586Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42793)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.528554567Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.529664542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.529698992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530282299Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530842496Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530862107Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[0]R flush_numbers[0..12]: [323, 322, 308, 319, 279, 318, 308, 320, 313, 308, 323, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530876897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[0]R generation[0..12]: [157, 156, 152, 156, 139, 155, 152, 156, 154, 152, 157, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530891517Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530906047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[1]R flush_numbers[0..12]: [323, 322, 308, 319, 279, 318, 308, 320, 313, 308, 323, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530920247Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[1]R generation[0..12]: [157, 156, 152, 156, 139, 155, 152, 156, 154, 152, 157, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530934367Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530948658Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[2]R flush_numbers[0..12]: [323, 322, 308, 319, 279, 318, 308, 320, 313, 308, 323, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530962758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[2]R generation[0..12]: [157, 156, 152, 156, 139, 155, 152, 156, 154, 152, 157, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530976838Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.530990978Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.531004519Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:58:22.531023679Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.531039389Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.531052699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"":"downstairs","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.531066249Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.531078429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"0","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.531091849Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"1","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.53110528Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"client":"2","":"downstairs","session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.53111885Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.53113123Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"81f1bf93-91ce-4e12-9324-14975a83cca5 is now active with session: 329fbac0-f4f2-493c-81a6-4bb969df25f9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.53114424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.53115628Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814,"session_id":"329fbac0-f4f2-493c-81a6-4bb969df25f9"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:22.531168621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 443 len 24576 data: 12 13 14 14 17 15 02/30 Read block 203 len 8192 03/30 Read block 256 len 24576 04/30 Read block 69 len 16384 05/30 Write block 377 len 28672 data: 22 21 18 16 13 13 8 06/30 Read block 284 len 20480 07/30 Flush 08/30 Write block 544 len 8192 data: 13 9 09/30 Read block 494 len 32768 10/30 Write block 547 len 28672 data: 8 11 14 16 16 14 12 11/30 Read block 352 len 40960 12/30 Read block 562 len 32768 13/30 Read block 4 len 32768 14/30 Read block 378 len 4096 15/30 Read block 542 len 24576 16/30 Read block 559 len 4096 17/30 Write block 18 len 4096 data: 15 18/30 Write block 517 len 4096 data: 8 19/30 Write block 324 len 24576 data: 15 14 15 13 12 12 20/30 Write block 184 len 20480 data: 5 4 6 8 9 21/30 Write block 536 len 4096 data: 11 22/30 Read block 58 len 20480 23/30 Flush 24/30 Write block 337 len 28672 data: 10 10 13 13 14 15 14 25/30 Read block 219 len 8192 26/30 Read block 426 len 40960 27/30 Read block 534 len 32768 28/30 Read block 564 len 24576 29/30 Read block 468 len 4096 30/30 Write block 404 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 1 Done Done Done false 31 Acked 1030 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:58:22.558457827Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 325 325 325 F F F 1 020-039 156 156 156 322 322 322 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 156 156 156 319 319 319 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 155 155 155 318 318 318 F F F 6 120-139 152 152 152 308 308 308 F F F 7 140-159 156 156 156 320 320 320 F F F 8 160-179 154 154 154 313 313 313 F F F 9 180-199 158 158 158 326 326 326 F F F 10 200-219 157 157 157 323 323 323 F F F 11 220-239 156 156 156 319 319 319 F F F 12 240-259 154 154 154 313 313 313 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 158 158 158 326 326 326 T T T 17 340-359 152 152 152 306 306 306 T T T 18 360-379 158 158 158 324 324 324 F F F 19 380-399 158 158 158 324 324 324 F F F 20 400-419 156 156 156 320 320 320 T T T 21 420-439 156 156 156 320 320 320 F F F 22 440-459 158 158 158 324 324 324 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 153 153 153 312 312 312 F F F 25 500-519 158 158 158 325 325 325 F F F 26 520-539 158 158 158 326 326 326 F F F 27 540-559 158 158 158 325 325 325 F F F 28 560-579 151 151 151 305 305 305 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 158, Max flush: 326 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 325 325 325 F F F 1 020-039 156 156 156 322 322 322 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 156 156 156 319 319 319 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 155 155 155 318 318 318 F F F 6 120-139 152 152 152 308 308 308 F F F 7 140-159 156 156 156 320 320 320 F F F 8 160-179 154 154 154 313 313 313 F F F 9 180-199 158 158 158 326 326 326 F F F 10 200-219 157 157 157 323 323 323 F F F 11 220-239 156 156 156 319 319 319 F F F 12 240-259 154 154 154 313 313 313 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 159 159 159 327 327 327 F F F 17 340-359 159 159 159 327 327 327 F F F 18 360-379 158 158 158 324 324 324 F F F 19 380-399 158 158 158 324 324 324 F F F 20 400-419 159 159 159 327 327 327 F F F 21 420-439 156 156 156 320 320 320 F F F 22 440-459 158 158 158 324 324 324 F F F 23 460-479 149 149 149 300 300 300 F F F 24 480-499 153 153 153 312 312 312 F F F 25 500-519 158 158 158 325 325 325 F F F 26 520-539 158 158 158 326 326 326 F F F 27 540-559 158 158 158 325 325 325 F F F 28 560-579 151 151 151 305 305 305 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 159, Max flush: 327 Begin loop 80 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.760155596Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.76044647Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.76047359Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822} {"msg":"Crucible c0006c41-94f7-448c-9f77-2eec8d97c5de has session id: dc52a92a-bb4b-43ba-823f-523feaefbf47","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.760832935Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Upstairs opts: Upstairs UUID: c0006c41-94f7-448c-9f77-2eec8d97c5de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.760873045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Crucible stats registered with UUID: c0006c41-94f7-448c-9f77-2eec8d97c5de","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.760889476Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761271601Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822} {"msg":"c0006c41-94f7-448c-9f77-2eec8d97c5de active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761302021Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761323891Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761345921Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761371312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761392842Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761414262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761437272Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761458733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761480883Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761502573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_connection connected from Ok(127.0.0.1:40627)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761739876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_connection connected from Ok(127.0.0.1:56926)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.761942439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:58:36.762083251Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:58:36.762116781Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:58:36.762148761Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.762171472Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.762186302Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.762217233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.762497916Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.762521866Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.762662828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.762690068Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.763111794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.763262176Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.76363608Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.763659331Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.763676281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.763781382Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.763802973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:36.763819853Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.763029743Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_connection connected from Ok(127.0.0.1:52201)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.763219455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.76436202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.76442855Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.764950787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765490984Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765514034Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[0]R flush_numbers[0..12]: [325, 322, 308, 319, 279, 318, 308, 320, 313, 326, 323, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765529295Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[0]R generation[0..12]: [158, 156, 152, 156, 139, 155, 152, 156, 154, 158, 157, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765543625Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765558245Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[1]R flush_numbers[0..12]: [325, 322, 308, 319, 279, 318, 308, 320, 313, 326, 323, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765572335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[1]R generation[0..12]: [158, 156, 152, 156, 139, 155, 152, 156, 154, 158, 157, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765586415Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765600565Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[2]R flush_numbers[0..12]: [325, 322, 308, 319, 279, 318, 308, 320, 313, 326, 323, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765614425Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[2]R generation[0..12]: [158, 156, 152, 156, 139, 155, 152, 156, 154, 158, 157, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765628326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765642376Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:58:46.765655416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765672536Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765685686Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765698737Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"":"downstairs","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765712047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765724097Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"0","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765737427Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"1","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765750577Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"client":"2","":"downstairs","session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765764047Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765776267Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"c0006c41-94f7-448c-9f77-2eec8d97c5de is now active with session: dc52a92a-bb4b-43ba-823f-523feaefbf47","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765789218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765801378Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822,"session_id":"dc52a92a-bb4b-43ba-823f-523feaefbf47"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:58:46.765813558Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 141 len 32768 data: 10 9 15 15 13 14 17 15 02/30 Read block 469 len 20480 03/30 Write block 480 len 28672 data: 11 11 10 10 11 10 11 04/30 Flush 05/30 Write block 143 len 32768 data: 16 16 14 15 18 16 15 12 06/30 Read block 53 len 36864 07/30 Read block 329 len 36864 08/30 Write block 206 len 12288 data: 11 13 13 09/30 Read block 143 len 20480 10/30 Read block 219 len 4096 11/30 Write block 165 len 40960 data: 9 11 12 11 10 8 7 8 12 13 12/30 Read block 289 len 20480 13/30 Read block 34 len 12288 14/30 Write block 247 len 16384 data: 15 16 16 16 15/30 Write block 344 len 24576 data: 14 13 13 15 15 11 16/30 Write block 135 len 12288 data: 17 15 18 17/30 Write block 128 len 4096 data: 11 18/30 Write block 462 len 28672 data: 11 12 15 15 13 12 11 19/30 Write block 572 len 12288 data: 10 9 12 20/30 Flush 21/30 Write block 179 len 28672 data: 10 8 6 5 5 6 5 22/30 Read block 561 len 8192 23/30 Write block 507 len 36864 data: 6 9 10 8 8 6 6 6 6 24/30 Write block 110 len 40960 data: 10 9 8 10 11 9 11 10 11 11 25/30 Read block 368 len 40960 26/30 Read block 217 len 24576 27/30 Write block 538 len 4096 data: 16 28/30 Read block 142 len 16384 29/30 Read block 253 len 28672 30/30 Write block 383 len 20480 data: 9 6 9 14 11 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 7 Done Done Done false 31 Acked 1030 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:58:46.793246146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 325 325 325 F F F 1 020-039 156 156 156 322 322 322 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 156 156 156 319 319 319 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 155 155 155 318 318 318 T T T 6 120-139 160 160 160 330 330 330 F F F 7 140-159 160 160 160 329 329 329 F F F 8 160-179 160 160 160 329 329 329 T T T 9 180-199 158 158 158 326 326 326 T T T 10 200-219 160 160 160 329 329 329 F F F 11 220-239 156 156 156 319 319 319 F F F 12 240-259 160 160 160 329 329 329 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 159 159 159 327 327 327 F F F 17 340-359 160 160 160 329 329 329 F F F 18 360-379 158 158 158 324 324 324 F F F 19 380-399 158 158 158 324 324 324 T T T 20 400-419 159 159 159 327 327 327 F F F 21 420-439 156 156 156 320 320 320 F F F 22 440-459 158 158 158 324 324 324 F F F 23 460-479 160 160 160 330 330 330 F F F 24 480-499 160 160 160 328 328 328 F F F 25 500-519 158 158 158 325 325 325 T T T 26 520-539 158 158 158 326 326 326 T T T 27 540-559 158 158 158 325 325 325 F F F 28 560-579 160 160 160 330 330 330 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 160, Max flush: 330 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 325 325 325 F F F 1 020-039 156 156 156 322 322 322 F F F 2 040-059 152 152 152 308 308 308 F F F 3 060-079 156 156 156 319 319 319 F F F 4 080-099 139 139 139 279 279 279 F F F 5 100-119 161 161 161 331 331 331 F F F 6 120-139 160 160 160 330 330 330 F F F 7 140-159 160 160 160 329 329 329 F F F 8 160-179 161 161 161 331 331 331 F F F 9 180-199 161 161 161 331 331 331 F F F 10 200-219 160 160 160 329 329 329 F F F 11 220-239 156 156 156 319 319 319 F F F 12 240-259 160 160 160 329 329 329 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 159 159 159 327 327 327 F F F 17 340-359 160 160 160 329 329 329 F F F 18 360-379 158 158 158 324 324 324 F F F 19 380-399 161 161 161 331 331 331 F F F 20 400-419 159 159 159 327 327 327 F F F 21 420-439 156 156 156 320 320 320 F F F 22 440-459 158 158 158 324 324 324 F F F 23 460-479 160 160 160 330 330 330 F F F 24 480-499 160 160 160 328 328 328 F F F 25 500-519 161 161 161 331 331 331 F F F 26 520-539 161 161 161 331 331 331 F F F 27 540-559 158 158 158 325 325 325 F F F 28 560-579 160 160 160 330 330 330 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 161, Max flush: 331 Begin loop 81 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.033747768Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034005881Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034023651Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830} {"msg":"Crucible e7710a90-63a1-487b-8dbc-171620fc8a3c has session id: b43df9a3-5f87-4739-9f31-79d88b13cff9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034341455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Upstairs opts: Upstairs UUID: e7710a90-63a1-487b-8dbc-171620fc8a3c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034388836Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Crucible stats registered with UUID: e7710a90-63a1-487b-8dbc-171620fc8a3c","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034413006Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.0347087Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830} {"msg":"e7710a90-63a1-487b-8dbc-171620fc8a3c active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034790651Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034813571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034829021Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034844422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034867742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034892552Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034917103Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034942073Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034967243Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.034990754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:01.035239147Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58159)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.035389859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:01.035419429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_connection connected from Ok(127.0.0.1:57747)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.035443439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:01.03546895Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.03549003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.03551726Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.03553899Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.035961536Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.035998996Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.036027647Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.036054867Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.036592034Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.036725196Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.03707692Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.037104101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.037119671Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.037193071Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.037213262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:01.037227292Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.036431652Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_connection connected from Ok(127.0.0.1:39908)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.036511163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.037512426Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.037540146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038104173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.03864836Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038670441Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[0]R flush_numbers[0..12]: [325, 322, 308, 319, 279, 331, 330, 329, 331, 331, 329, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038685401Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[0]R generation[0..12]: [158, 156, 152, 156, 139, 161, 160, 160, 161, 161, 160, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038700031Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038714651Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[1]R flush_numbers[0..12]: [325, 322, 308, 319, 279, 331, 330, 329, 331, 331, 329, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038728731Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[1]R generation[0..12]: [158, 156, 152, 156, 139, 161, 160, 160, 161, 161, 160, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038742791Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038757012Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[2]R flush_numbers[0..12]: [325, 322, 308, 319, 279, 331, 330, 329, 331, 331, 329, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038770892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[2]R generation[0..12]: [158, 156, 152, 156, 139, 161, 160, 160, 161, 161, 160, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038784742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038803542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:59:11.038817042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038831023Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038844093Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038857193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"":"downstairs","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038870903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038882953Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"0","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038896563Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"1","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038910133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"client":"2","":"downstairs","session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038923774Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038935964Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"e7710a90-63a1-487b-8dbc-171620fc8a3c is now active with session: b43df9a3-5f87-4739-9f31-79d88b13cff9","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038955684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038975354Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830,"session_id":"b43df9a3-5f87-4739-9f31-79d88b13cff9"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:11.038997615Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 483 len 28672 data: 11 12 11 12 13 13 12 02/30 Write block 83 len 16384 data: 9 8 9 8 03/30 Write block 440 len 24576 data: 10 15 13 13 14 15 04/30 Write block 44 len 4096 data: 15 05/30 Write block 479 len 24576 data: 13 12 12 11 12 13 06/30 Write block 63 len 16384 data: 10 12 15 13 07/30 Read block 250 len 4096 08/30 Read block 337 len 8192 09/30 Read block 422 len 16384 10/30 Write block 9 len 16384 data: 11 9 11 6 11/30 Read block 158 len 12288 12/30 Write block 377 len 32768 data: 23 22 19 17 14 14 10 7 13/30 Write block 554 len 24576 data: 11 10 7 10 8 5 14/30 Read block 584 len 12288 15/30 Read block 524 len 8192 16/30 Write block 469 len 4096 data: 10 17/30 Write block 21 len 28672 data: 13 12 10 11 11 11 10 18/30 Read block 20 len 12288 19/30 Flush 20/30 Write block 363 len 40960 data: 8 6 7 8 6 10 13 16 16 13 21/30 Read block 288 len 8192 22/30 Read block 519 len 36864 23/30 Read block 445 len 20480 24/30 Read block 230 len 36864 25/30 Write block 465 len 12288 data: 16 14 13 26/30 Write block 83 len 28672 data: 10 9 10 9 11 10 10 27/30 Write block 338 len 16384 data: 11 14 14 15 28/30 Read block 184 len 28672 29/30 Read block 527 len 8192 30/30 Read block 340 len 32768 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 10 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:11.064910514Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 332 332 332 F F F 1 020-039 162 162 162 332 332 332 F F F 2 040-059 162 162 162 332 332 332 F F F 3 060-079 162 162 162 332 332 332 F F F 4 080-099 162 162 162 332 332 332 T T T 5 100-119 161 161 161 331 331 331 F F F 6 120-139 160 160 160 330 330 330 F F F 7 140-159 160 160 160 329 329 329 F F F 8 160-179 161 161 161 331 331 331 F F F 9 180-199 161 161 161 331 331 331 F F F 10 200-219 160 160 160 329 329 329 F F F 11 220-239 156 156 156 319 319 319 F F F 12 240-259 160 160 160 329 329 329 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 159 159 159 327 327 327 T T T 17 340-359 160 160 160 329 329 329 T T T 18 360-379 162 162 162 332 332 332 T T T 19 380-399 162 162 162 332 332 332 F F F 20 400-419 159 159 159 327 327 327 F F F 21 420-439 156 156 156 320 320 320 F F F 22 440-459 162 162 162 332 332 332 F F F 23 460-479 162 162 162 332 332 332 T T T 24 480-499 162 162 162 332 332 332 F F F 25 500-519 161 161 161 331 331 331 F F F 26 520-539 161 161 161 331 331 331 F F F 27 540-559 162 162 162 332 332 332 F F F 28 560-579 160 160 160 330 330 330 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 162, Max flush: 332 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 332 332 332 F F F 1 020-039 162 162 162 332 332 332 F F F 2 040-059 162 162 162 332 332 332 F F F 3 060-079 162 162 162 332 332 332 F F F 4 080-099 163 163 163 333 333 333 F F F 5 100-119 161 161 161 331 331 331 F F F 6 120-139 160 160 160 330 330 330 F F F 7 140-159 160 160 160 329 329 329 F F F 8 160-179 161 161 161 331 331 331 F F F 9 180-199 161 161 161 331 331 331 F F F 10 200-219 160 160 160 329 329 329 F F F 11 220-239 156 156 156 319 319 319 F F F 12 240-259 160 160 160 329 329 329 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 154 154 154 317 317 317 F F F 16 320-339 163 163 163 333 333 333 F F F 17 340-359 163 163 163 333 333 333 F F F 18 360-379 163 163 163 333 333 333 F F F 19 380-399 162 162 162 332 332 332 F F F 20 400-419 159 159 159 327 327 327 F F F 21 420-439 156 156 156 320 320 320 F F F 22 440-459 162 162 162 332 332 332 F F F 23 460-479 163 163 163 333 333 333 F F F 24 480-499 162 162 162 332 332 332 F F F 25 500-519 161 161 161 331 331 331 F F F 26 520-539 161 161 161 331 331 331 F F F 27 540-559 162 162 162 332 332 332 F F F 28 560-579 160 160 160 330 330 330 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 163, Max flush: 333 Begin loop 82 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.291690241Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.29238685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.292416561Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838} {"msg":"Crucible 045414eb-aa53-4fe5-b6b2-35005ba91bf0 has session id: 715ca2b3-bf7f-4931-86f8-2bbd62a2f55e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.292490211Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Upstairs opts: Upstairs UUID: 045414eb-aa53-4fe5-b6b2-35005ba91bf0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.292525572Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Crucible stats registered with UUID: 045414eb-aa53-4fe5-b6b2-35005ba91bf0","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.292547382Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.292882696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838} {"msg":"045414eb-aa53-4fe5-b6b2-35005ba91bf0 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.292945987Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.292972468Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.292988458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293002628Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293017018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293030509Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293054119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293075419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293090059Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293103809Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:25.293309222Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45162)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293347413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:25.293377713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44004)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293404053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:25.293566695Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293583196Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293597136Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.293610526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.294074852Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.294100262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.294114692Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.294128342Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.29472884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.2947567Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.295272977Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.295299887Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.295314158Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.295339638Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.295353998Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:25.295372508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.295225166Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61147)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.295460569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.296451262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.296475032Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.29713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297743528Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297770748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[0]R flush_numbers[0..12]: [332, 332, 332, 332, 333, 331, 330, 329, 331, 331, 329, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297788989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[0]R generation[0..12]: [162, 162, 162, 162, 163, 161, 160, 160, 161, 161, 160, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297806799Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297825059Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[1]R flush_numbers[0..12]: [332, 332, 332, 332, 333, 331, 330, 329, 331, 331, 329, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297842799Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[1]R generation[0..12]: [162, 162, 162, 162, 163, 161, 160, 160, 161, 161, 160, 156]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.29786037Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.29787829Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[2]R flush_numbers[0..12]: [332, 332, 332, 332, 333, 331, 330, 329, 331, 331, 329, 319]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.29789572Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[2]R generation[0..12]: [162, 162, 162, 162, 163, 161, 160, 160, 161, 161, 160, 156]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T15:59:35.29791858Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297938471Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297955111Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297971791Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.297988091Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298004461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"":"downstairs","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298021081Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298036102Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"0","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298052712Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"1","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298069152Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"client":"2","":"downstairs","session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298085932Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298101353Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"045414eb-aa53-4fe5-b6b2-35005ba91bf0 is now active with session: 715ca2b3-bf7f-4931-86f8-2bbd62a2f55e","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298117513Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298132453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838,"session_id":"715ca2b3-bf7f-4931-86f8-2bbd62a2f55e"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:35.298147713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 119 len 32768 data: 12 12 10 11 9 10 11 10 02/30 Read block 370 len 20480 03/30 Flush 04/30 Write block 246 len 28672 data: 16 16 17 17 17 17 15 05/30 Read block 102 len 32768 06/30 Read block 177 len 20480 07/30 Read block 226 len 28672 08/30 Read block 276 len 8192 09/30 Write block 246 len 24576 data: 17 17 18 18 18 18 10/30 Write block 85 len 28672 data: 11 10 12 11 11 10 10 11/30 Read block 165 len 36864 12/30 Read block 108 len 32768 13/30 Write block 446 len 8192 data: 15 18 14/30 Read block 561 len 32768 15/30 Read block 106 len 40960 16/30 Read block 479 len 20480 17/30 Read block 405 len 40960 18/30 Read block 583 len 36864 19/30 Flush 20/30 Write block 3 len 24576 data: 12 14 16 15 15 15 21/30 Read block 65 len 20480 22/30 Write block 301 len 36864 data: 9 11 12 12 14 14 13 11 10 23/30 Read block 271 len 24576 24/30 Write block 378 len 28672 data: 23 20 18 15 15 11 8 25/30 Read block 3 len 32768 26/30 Read block 308 len 8192 27/30 Write block 215 len 32768 data: 10 10 8 9 11 12 17 17 28/30 Read block 121 len 40960 29/30 Read block 264 len 12288 30/30 Write block 69 len 40960 data: 12 14 14 13 16 13 14 14 14 11 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false 31 Acked 1030 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:35.325584661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 332 332 332 T T T 1 020-039 162 162 162 332 332 332 F F F 2 040-059 162 162 162 332 332 332 F F F 3 060-079 162 162 162 332 332 332 T T T 4 080-099 164 164 164 335 335 335 F F F 5 100-119 164 164 164 334 334 334 F F F 6 120-139 164 164 164 334 334 334 F F F 7 140-159 160 160 160 329 329 329 F F F 8 160-179 161 161 161 331 331 331 F F F 9 180-199 161 161 161 331 331 331 F F F 10 200-219 160 160 160 329 329 329 T T T 11 220-239 156 156 156 319 319 319 T T T 12 240-259 164 164 164 335 335 335 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 154 154 154 317 317 317 T T T 16 320-339 163 163 163 333 333 333 F F F 17 340-359 163 163 163 333 333 333 F F F 18 360-379 163 163 163 333 333 333 T T T 19 380-399 162 162 162 332 332 332 T T T 20 400-419 159 159 159 327 327 327 F F F 21 420-439 156 156 156 320 320 320 F F F 22 440-459 164 164 164 335 335 335 F F F 23 460-479 163 163 163 333 333 333 F F F 24 480-499 162 162 162 332 332 332 F F F 25 500-519 161 161 161 331 331 331 F F F 26 520-539 161 161 161 331 331 331 F F F 27 540-559 162 162 162 332 332 332 F F F 28 560-579 160 160 160 330 330 330 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 164, Max flush: 335 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 162 162 162 332 332 332 F F F 2 040-059 162 162 162 332 332 332 F F F 3 060-079 165 165 165 336 336 336 F F F 4 080-099 164 164 164 335 335 335 F F F 5 100-119 164 164 164 334 334 334 F F F 6 120-139 164 164 164 334 334 334 F F F 7 140-159 160 160 160 329 329 329 F F F 8 160-179 161 161 161 331 331 331 F F F 9 180-199 161 161 161 331 331 331 F F F 10 200-219 165 165 165 336 336 336 F F F 11 220-239 165 165 165 336 336 336 F F F 12 240-259 164 164 164 335 335 335 F F F 13 260-279 155 155 155 318 318 318 F F F 14 280-299 156 156 156 321 321 321 F F F 15 300-319 165 165 165 336 336 336 F F F 16 320-339 163 163 163 333 333 333 F F F 17 340-359 163 163 163 333 333 333 F F F 18 360-379 165 165 165 336 336 336 F F F 19 380-399 165 165 165 336 336 336 F F F 20 400-419 159 159 159 327 327 327 F F F 21 420-439 156 156 156 320 320 320 F F F 22 440-459 164 164 164 335 335 335 F F F 23 460-479 163 163 163 333 333 333 F F F 24 480-499 162 162 162 332 332 332 F F F 25 500-519 161 161 161 331 331 331 F F F 26 520-539 161 161 161 331 331 331 F F F 27 540-559 162 162 162 332 332 332 F F F 28 560-579 160 160 160 330 330 330 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 165, Max flush: 336 Begin loop 83 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.581072874Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.581645481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.581674071Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846} {"msg":"Crucible 1109acad-7a45-4b03-99e1-35b5ed12e1ab has session id: 79622311-906b-49be-ab21-cb2bab07a3cf","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.581914934Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Upstairs opts: Upstairs UUID: 1109acad-7a45-4b03-99e1-35b5ed12e1ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.581963195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Crucible stats registered with UUID: 1109acad-7a45-4b03-99e1-35b5ed12e1ab","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.581996065Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582287129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846} {"msg":"1109acad-7a45-4b03-99e1-35b5ed12e1ab active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.58233295Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.58235098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.58236595Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.5823841Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582408441Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582434311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582460911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582486142Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582511332Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582535212Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:49.582647104Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:49.582689024Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:61792)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582891477Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:49.582932077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582953568Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582975278Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:49157)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.582997458Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.583020728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.583505524Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.583543845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.583573955Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.583601306Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.584186383Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.584225074Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.58474522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.584777701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.584796741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.584812921Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.584834941Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:49.584851001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.584040871Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:42078)","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.584211793Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.585309267Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.585348508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.585955166Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586502673Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586527363Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[0]R flush_numbers[0..12]: [336, 332, 332, 336, 335, 334, 334, 329, 331, 331, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586546293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[0]R generation[0..12]: [165, 162, 162, 165, 164, 164, 164, 160, 161, 161, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586564673Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586582754Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[1]R flush_numbers[0..12]: [336, 332, 332, 336, 335, 334, 334, 329, 331, 331, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586600324Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[1]R generation[0..12]: [165, 162, 162, 165, 164, 164, 164, 160, 161, 161, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586617794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586640694Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[2]R flush_numbers[0..12]: [336, 332, 332, 336, 335, 334, 334, 329, 331, 331, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586660275Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[2]R generation[0..12]: [165, 162, 162, 165, 164, 164, 164, 160, 161, 161, 165, 165]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-07-10T15:59:59.586680655Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586698395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586714855Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586731516Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586747826Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586764276Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"":"downstairs","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586781056Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586796326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"0","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586812966Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"1","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586829557Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"client":"2","":"downstairs","session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586846487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586861857Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"1109acad-7a45-4b03-99e1-35b5ed12e1ab is now active with session: 79622311-906b-49be-ab21-cb2bab07a3cf","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586877907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586893007Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846,"session_id":"79622311-906b-49be-ab21-cb2bab07a3cf"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-07-10T15:59:59.586910758Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 208 len 8192 data: 14 15 02/30 Read block 593 len 24576 03/30 Write block 46 len 28672 data: 12 10 10 11 10 11 14 04/30 Write block 294 len 20480 data: 13 10 10 10 8 05/30 Read block 533 len 4096 06/30 Write block 352 len 28672 data: 12 13 13 14 15 12 14 07/30 Write block 90 len 8192 data: 11 11 08/30 Read block 224 len 28672 09/30 Write block 254 len 40960 data: 18 20 18 15 11 13 14 11 11 11 10/30 Write block 58 len 36864 data: 11 13 12 14 10 11 13 16 14 11/30 Read block 203 len 40960 12/30 Read block 410 len 16384 13/30 Flush 14/30 Write block 391 len 28672 data: 16 15 17 15 15 17 19 15/30 Write block 472 len 24576 data: 8 11 11 15 18 16 16/30 Write block 422 len 36864 data: 10 10 13 13 13 15 15 16 14 17/30 Write block 322 len 20480 data: 16 16 16 15 16 18/30 Write block 195 len 32768 data: 11 14 15 13 11 11 9 9 19/30 Write block 189 len 36864 data: 12 11 15 16 14 13 12 15 16 20/30 Write block 510 len 40960 data: 9 9 7 7 7 7 7 9 8 7 21/30 Read block 596 len 8192 22/30 Write block 228 len 4096 data: 14 23/30 Write block 22 len 36864 data: 13 11 12 12 12 11 11 13 9 24/30 Flush 25/30 Write block 557 len 32768 data: 11 9 6 6 7 10 8 10 26/30 Write block 397 len 24576 data: 20 18 18 17 14 13 27/30 Read block 78 len 24576 28/30 Write block 528 len 36864 data: 18 19 19 20 18 11 12 10 12 29/30 Read block 270 len 36864 30/30 Read block 514 len 16384 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 9 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false 31 Acked 1030 Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T15:59:59.617125481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 166 166 166 337 337 337 F F F 3 060-079 166 166 166 337 337 337 F F F 4 080-099 166 166 166 337 337 337 F F F 5 100-119 164 164 164 334 334 334 F F F 6 120-139 164 164 164 334 334 334 F F F 7 140-159 160 160 160 329 329 329 F F F 8 160-179 161 161 161 331 331 331 F F F 9 180-199 166 166 166 338 338 338 F F F 10 200-219 166 166 166 338 338 338 F F F 11 220-239 166 166 166 339 339 339 F F F 12 240-259 166 166 166 337 337 337 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 166 166 166 337 337 337 F F F 15 300-319 165 165 165 336 336 336 F F F 16 320-339 166 166 166 338 338 338 F F F 17 340-359 166 166 166 337 337 337 F F F 18 360-379 165 165 165 336 336 336 F F F 19 380-399 166 166 166 338 338 338 T T T 20 400-419 159 159 159 327 327 327 T T T 21 420-439 166 166 166 338 338 338 F F F 22 440-459 164 164 164 335 335 335 F F F 23 460-479 166 166 166 338 338 338 F F F 24 480-499 162 162 162 332 332 332 F F F 25 500-519 166 166 166 339 339 339 F F F 26 520-539 161 161 161 331 331 331 T T T 27 540-559 162 162 162 332 332 332 T T T 28 560-579 160 160 160 330 330 330 T T T 29 580-599 151 151 151 305 305 305 F F F Max gen: 166, Max flush: 339 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 166 166 166 337 337 337 F F F 3 060-079 166 166 166 337 337 337 F F F 4 080-099 166 166 166 337 337 337 F F F 5 100-119 164 164 164 334 334 334 F F F 6 120-139 164 164 164 334 334 334 F F F 7 140-159 160 160 160 329 329 329 F F F 8 160-179 161 161 161 331 331 331 F F F 9 180-199 166 166 166 338 338 338 F F F 10 200-219 166 166 166 338 338 338 F F F 11 220-239 166 166 166 339 339 339 F F F 12 240-259 166 166 166 337 337 337 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 166 166 166 337 337 337 F F F 15 300-319 165 165 165 336 336 336 F F F 16 320-339 166 166 166 338 338 338 F F F 17 340-359 166 166 166 337 337 337 F F F 18 360-379 165 165 165 336 336 336 F F F 19 380-399 167 167 167 340 340 340 F F F 20 400-419 167 167 167 340 340 340 F F F 21 420-439 166 166 166 338 338 338 F F F 22 440-459 164 164 164 335 335 335 F F F 23 460-479 166 166 166 338 338 338 F F F 24 480-499 162 162 162 332 332 332 F F F 25 500-519 166 166 166 339 339 339 F F F 26 520-539 167 167 167 340 340 340 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 167 167 167 340 340 340 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 167, Max flush: 340 Begin loop 84 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.843156429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.843400262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.843418933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854} {"msg":"Crucible fe6f15d5-4aff-4f25-858a-e46804b4d532 has session id: 0c137aa8-1ed3-4814-873e-37414e061dc0","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.843825248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Upstairs opts: Upstairs UUID: fe6f15d5-4aff-4f25-858a-e46804b4d532, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.843872798Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Crucible stats registered with UUID: fe6f15d5-4aff-4f25-858a-e46804b4d532","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.843897219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844220083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854} {"msg":"fe6f15d5-4aff-4f25-858a-e46804b4d532 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844244343Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844258333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844273053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844286644Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844306994Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844330794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844357564Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844381845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844405665Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844429525Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36153)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844599417Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:00:13.844636928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:00:13.844672508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:00:13.844702419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844723349Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844748369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36319)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.8447746Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.844834111Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.845358447Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.845382907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.845397978Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.845414028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.846052326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.846079496Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.846575133Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.846599733Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.846617193Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.846642713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.846656823Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:13.846670204Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.846022395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61370)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.846230988Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.847361712Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.847417993Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.84794452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848480157Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848501547Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[0]R flush_numbers[0..12]: [336, 339, 337, 337, 337, 334, 334, 329, 331, 338, 338, 339]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848516937Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[0]R generation[0..12]: [165, 166, 166, 166, 166, 164, 164, 160, 161, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848531487Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848545927Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[1]R flush_numbers[0..12]: [336, 339, 337, 337, 337, 334, 334, 329, 331, 338, 338, 339]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848560028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[1]R generation[0..12]: [165, 166, 166, 166, 166, 164, 164, 160, 161, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848578478Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848593248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[2]R flush_numbers[0..12]: [336, 339, 337, 337, 337, 334, 334, 329, 331, 338, 338, 339]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848610608Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[2]R generation[0..12]: [165, 166, 166, 166, 166, 164, 164, 160, 161, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848627839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T16:00:23.848645569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848662179Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848678859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848695369Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.84871175Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"":"downstairs","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.84873857Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.8487674Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"0","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848790971Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"1","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848813701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"client":"2","":"downstairs","session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848842601Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848870202Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"fe6f15d5-4aff-4f25-858a-e46804b4d532 is now active with session: 0c137aa8-1ed3-4814-873e-37414e061dc0","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848906242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-07-10T16:00:23.848933532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854,"session_id":"0c137aa8-1ed3-4814-873e-37414e061dc0"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:23.848961233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 334 len 4096 data: 13 02/30 Write block 198 len 20480 data: 14 12 12 10 10 03/30 Read block 289 len 16384 04/30 Write block 300 len 4096 data: 8 05/30 Write block 99 len 8192 data: 12 14 06/30 Read block 308 len 12288 07/30 Write block 41 len 16384 data: 11 12 15 16 08/30 Read block 343 len 24576 09/30 Read block 590 len 20480 10/30 Flush 11/30 Write block 330 len 40960 data: 12 12 14 11 14 10 10 11 12 15 12/30 Write block 437 len 12288 data: 10 9 11 13/30 Read block 312 len 24576 14/30 Read block 297 len 20480 15/30 Read block 367 len 20480 16/30 Read block 227 len 24576 17/30 Read block 526 len 8192 18/30 Write block 76 len 36864 data: 15 15 12 10 8 7 8 11 10 19/30 Read block 504 len 28672 20/30 Flush 21/30 Write block 459 len 40960 data: 6 8 10 12 13 16 17 15 14 12 22/30 Write block 141 len 36864 data: 11 10 17 17 15 16 19 17 16 23/30 Read block 143 len 8192 24/30 Read block 375 len 4096 25/30 Flush 26/30 Write block 42 len 28672 data: 13 16 17 12 13 11 11 27/30 Write block 402 len 20480 data: 14 14 13 11 10 28/30 Write block 521 len 36864 data: 7 7 8 11 12 18 17 19 20 29/30 Read block 261 len 40960 30/30 Read block 276 len 8192 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:00:23.876500872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 168 168 168 341 341 341 T T T 3 060-079 168 168 168 342 342 342 F F F 4 080-099 168 168 168 342 342 342 F F F 5 100-119 168 168 168 341 341 341 F F F 6 120-139 164 164 164 334 334 334 F F F 7 140-159 168 168 168 343 343 343 F F F 8 160-179 161 161 161 331 331 331 F F F 9 180-199 168 168 168 341 341 341 F F F 10 200-219 168 168 168 341 341 341 F F F 11 220-239 166 166 166 339 339 339 F F F 12 240-259 166 166 166 337 337 337 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 166 166 166 337 337 337 F F F 15 300-319 168 168 168 341 341 341 F F F 16 320-339 168 168 168 342 342 342 F F F 17 340-359 166 166 166 337 337 337 F F F 18 360-379 165 165 165 336 336 336 F F F 19 380-399 167 167 167 340 340 340 F F F 20 400-419 167 167 167 340 340 340 T T T 21 420-439 168 168 168 342 342 342 F F F 22 440-459 168 168 168 343 343 343 F F F 23 460-479 168 168 168 343 343 343 F F F 24 480-499 162 162 162 332 332 332 F F F 25 500-519 166 166 166 339 339 339 F F F 26 520-539 167 167 167 340 340 340 T T T 27 540-559 167 167 167 340 340 340 F F F 28 560-579 167 167 167 340 340 340 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 168, Max flush: 343 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 169 169 169 344 344 344 F F F 3 060-079 168 168 168 342 342 342 F F F 4 080-099 168 168 168 342 342 342 F F F 5 100-119 168 168 168 341 341 341 F F F 6 120-139 164 164 164 334 334 334 F F F 7 140-159 168 168 168 343 343 343 F F F 8 160-179 161 161 161 331 331 331 F F F 9 180-199 168 168 168 341 341 341 F F F 10 200-219 168 168 168 341 341 341 F F F 11 220-239 166 166 166 339 339 339 F F F 12 240-259 166 166 166 337 337 337 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 166 166 166 337 337 337 F F F 15 300-319 168 168 168 341 341 341 F F F 16 320-339 168 168 168 342 342 342 F F F 17 340-359 166 166 166 337 337 337 F F F 18 360-379 165 165 165 336 336 336 F F F 19 380-399 167 167 167 340 340 340 F F F 20 400-419 169 169 169 344 344 344 F F F 21 420-439 168 168 168 342 342 342 F F F 22 440-459 168 168 168 343 343 343 F F F 23 460-479 168 168 168 343 343 343 F F F 24 480-499 162 162 162 332 332 332 F F F 25 500-519 166 166 166 339 339 339 F F F 26 520-539 169 169 169 344 344 344 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 167 167 167 340 340 340 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 169, Max flush: 344 Begin loop 85 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.079203015Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.07960887Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.079637491Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862} {"msg":"Crucible e82312c6-47e6-48a0-8129-860b8e492174 has session id: 6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.079825593Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Upstairs opts: Upstairs UUID: e82312c6-47e6-48a0-8129-860b8e492174, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.079871863Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Crucible stats registered with UUID: e82312c6-47e6-48a0-8129-860b8e492174","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.079895304Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080234098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862} {"msg":"e82312c6-47e6-48a0-8129-860b8e492174 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080266479Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080289739Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080315149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080339089Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.08036447Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.08038854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.08041371Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080437281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080462681Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080487511Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46056)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080609233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37928)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080660264Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:00:38.080762755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:00:38.080792335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:00:38.080916587Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080935027Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.080948957Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.081082419Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.081347032Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.081384693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.081558025Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.081595305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.081926749Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.082045781Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.082489597Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.082513977Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.082533317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.082546297Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.082559868Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:38.082573318Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.08274906Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37388)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.082886372Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.084000396Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.084033416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.084546073Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085073309Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.08510421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[0]R flush_numbers[0..12]: [336, 339, 344, 342, 342, 341, 334, 343, 331, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.08512371Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[0]R generation[0..12]: [165, 166, 169, 168, 168, 168, 164, 168, 161, 168, 168, 166]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.08514248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.08516051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[1]R flush_numbers[0..12]: [336, 339, 344, 342, 342, 341, 334, 343, 331, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085179071Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[1]R generation[0..12]: [165, 166, 169, 168, 168, 168, 164, 168, 161, 168, 168, 166]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085197771Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085212451Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[2]R flush_numbers[0..12]: [336, 339, 344, 342, 342, 341, 334, 343, 331, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085226742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[2]R generation[0..12]: [165, 166, 169, 168, 168, 168, 164, 168, 161, 168, 168, 166]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-07-10T16:00:48.085245082Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085259602Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085272832Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085286242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085299312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085312403Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"":"downstairs","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085325773Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085337783Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"0","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085351213Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"1","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085364573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"client":"2","":"downstairs","session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085378243Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085390633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"e82312c6-47e6-48a0-8129-860b8e492174 is now active with session: 6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085405504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085417844Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862,"session_id":"6513f1b6-21a5-4af8-ac0d-3ff8422c4b0e"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:00:48.085430134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 504 len 12288 data: 12 10 9 02/30 Write block 331 len 20480 data: 13 15 12 15 11 03/30 Write block 199 len 36864 data: 13 13 11 11 11 12 10 12 14 04/30 Read block 145 len 32768 05/30 Flush 06/30 Write block 133 len 28672 data: 15 17 18 16 19 19 19 07/30 Read block 408 len 24576 08/30 Read block 107 len 36864 09/30 Read block 182 len 4096 10/30 Flush 11/30 Write block 348 len 40960 data: 16 12 11 11 13 14 14 15 16 13 12/30 Flush 13/30 Write block 496 len 20480 data: 12 13 14 14 14 14/30 Read block 15 len 12288 15/30 Read block 88 len 12288 16/30 Read block 268 len 32768 17/30 Read block 199 len 32768 18/30 Read block 327 len 8192 19/30 Write block 578 len 8192 data: 16 13 20/30 Read block 174 len 20480 21/30 Write block 355 len 8192 data: 16 17 22/30 Read block 223 len 40960 23/30 Flush 24/30 Write block 159 len 16384 data: 13 9 10 9 25/30 Read block 97 len 24576 26/30 Write block 483 len 20480 data: 13 14 12 13 14 27/30 Write block 430 len 4096 data: 15 28/30 Write block 296 len 8192 data: 11 11 29/30 Read block 252 len 28672 30/30 Read block 102 len 32768 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:00:48.114003407Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 169 169 169 344 344 344 F F F 3 060-079 168 168 168 342 342 342 F F F 4 080-099 168 168 168 342 342 342 F F F 5 100-119 168 168 168 341 341 341 F F F 6 120-139 170 170 170 346 346 346 F F F 7 140-159 168 168 168 343 343 343 T T T 8 160-179 161 161 161 331 331 331 T T T 9 180-199 170 170 170 345 345 345 F F F 10 200-219 170 170 170 345 345 345 F F F 11 220-239 166 166 166 339 339 339 F F F 12 240-259 166 166 166 337 337 337 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 166 166 166 337 337 337 T T T 15 300-319 168 168 168 341 341 341 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 170 170 170 348 348 348 F F F 18 360-379 165 165 165 336 336 336 F F F 19 380-399 167 167 167 340 340 340 F F F 20 400-419 169 169 169 344 344 344 F F F 21 420-439 168 168 168 342 342 342 T T T 22 440-459 168 168 168 343 343 343 F F F 23 460-479 168 168 168 343 343 343 F F F 24 480-499 170 170 170 348 348 348 T T T 25 500-519 170 170 170 348 348 348 F F F 26 520-539 169 169 169 344 344 344 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 170 170 170 348 348 348 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 170, Max flush: 348 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 169 169 169 344 344 344 F F F 3 060-079 168 168 168 342 342 342 F F F 4 080-099 168 168 168 342 342 342 F F F 5 100-119 168 168 168 341 341 341 F F F 6 120-139 170 170 170 346 346 346 F F F 7 140-159 171 171 171 349 349 349 F F F 8 160-179 171 171 171 349 349 349 F F F 9 180-199 170 170 170 345 345 345 F F F 10 200-219 170 170 170 345 345 345 F F F 11 220-239 166 166 166 339 339 339 F F F 12 240-259 166 166 166 337 337 337 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 171 171 171 349 349 349 F F F 15 300-319 168 168 168 341 341 341 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 170 170 170 348 348 348 F F F 18 360-379 165 165 165 336 336 336 F F F 19 380-399 167 167 167 340 340 340 F F F 20 400-419 169 169 169 344 344 344 F F F 21 420-439 171 171 171 349 349 349 F F F 22 440-459 168 168 168 343 343 343 F F F 23 460-479 168 168 168 343 343 343 F F F 24 480-499 171 171 171 349 349 349 F F F 25 500-519 170 170 170 348 348 348 F F F 26 520-539 169 169 169 344 344 344 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 170 170 170 348 348 348 F F F 29 580-599 151 151 151 305 305 305 F F F Max gen: 171, Max flush: 349 Begin loop 86 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.337164648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.337655914Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.337687575Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870} {"msg":"Crucible d157d12f-54d6-4902-abd9-13cddb573201 has session id: 7098f77e-edd2-4fae-8f0c-1a5df7732e6d","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.337836227Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Upstairs opts: Upstairs UUID: d157d12f-54d6-4902-abd9-13cddb573201, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.337887597Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Crucible stats registered with UUID: d157d12f-54d6-4902-abd9-13cddb573201","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.337912508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338361654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870} {"msg":"d157d12f-54d6-4902-abd9-13cddb573201 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338386384Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338400794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338422454Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338447505Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338472765Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338495615Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338519265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338543526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338566356Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.338589497Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:02.33884696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:02.338967351Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40590)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.339001682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:02.339030962Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.339046312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.339065463Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60239)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.339082263Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.339096413Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.339541709Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.339569019Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.339585549Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.339609299Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.340152806Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.340287358Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.340680573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.340710833Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.340731313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.340744954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.340759394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:02.340772684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.340678503Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:37796)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.340917036Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.341861278Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.341901928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.342451995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343012742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343038333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[0]R flush_numbers[0..12]: [336, 339, 344, 342, 342, 341, 346, 349, 349, 345, 345, 339]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343057303Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[0]R generation[0..12]: [165, 166, 169, 168, 168, 168, 170, 171, 171, 170, 170, 166]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343075503Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343093543Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[1]R flush_numbers[0..12]: [336, 339, 344, 342, 342, 341, 346, 349, 349, 345, 345, 339]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343110954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[1]R generation[0..12]: [165, 166, 169, 168, 168, 168, 170, 171, 171, 170, 170, 166]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343133544Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343151604Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[2]R flush_numbers[0..12]: [336, 339, 344, 342, 342, 341, 346, 349, 349, 345, 345, 339]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343169175Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"[2]R generation[0..12]: [165, 166, 169, 168, 168, 168, 170, 171, 171, 170, 170, 166]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343186545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343207565Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343224635Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343241625Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Next flush: 350","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343258016Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343274466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"":"downstairs","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343290926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343306146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"0","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343323146Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"1","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343339747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"client":"2","":"downstairs","session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343356547Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343371897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"d157d12f-54d6-4902-abd9-13cddb573201 is now active with session: 7098f77e-edd2-4fae-8f0c-1a5df7732e6d","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343391317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:12.343406667Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870,"session_id":"7098f77e-edd2-4fae-8f0c-1a5df7732e6d"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 30,"time":"2024-07-10T16:01:12.343426308Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 151 len 4096 data: 11 02/30 Read block 215 len 12288 03/30 Read block 293 len 4096 04/30 Write block 289 len 4096 data: 11 05/30 Flush 06/30 Write block 374 len 24576 data: 18 15 17 24 24 21 07/30 Read block 300 len 40960 08/30 Write block 389 len 36864 data: 16 19 17 16 18 16 16 18 21 09/30 Read block 24 len 16384 10/30 Write block 391 len 4096 data: 18 11/30 Read block 177 len 12288 12/30 Read block 47 len 40960 13/30 Read block 376 len 40960 14/30 Read block 102 len 4096 15/30 Write block 246 len 12288 data: 18 18 19 16/30 Flush 17/30 Write block 237 len 28672 data: 14 12 13 12 15 15 16 18/30 Read block 477 len 40960 19/30 Write block 592 len 8192 data: 12 8 20/30 Read block 186 len 40960 21/30 Read block 121 len 16384 22/30 Read block 469 len 32768 23/30 Write block 286 len 24576 data: 8 10 10 12 10 10 24/30 Write block 386 len 16384 data: 15 12 14 17 25/30 Read block 74 len 32768 26/30 Write block 78 len 28672 data: 13 11 9 8 9 12 11 27/30 Read block 7 len 8192 28/30 Read block 177 len 24576 29/30 Write block 527 len 4096 data: 18 30/30 Read block 24 len 16384 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 1 Done Done Done false 31 Acked 1030 Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:12.369147534Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 169 169 169 344 344 344 F F F 3 060-079 168 168 168 342 342 342 T T T 4 080-099 168 168 168 342 342 342 T T T 5 100-119 168 168 168 341 341 341 F F F 6 120-139 170 170 170 346 346 346 F F F 7 140-159 172 172 172 350 350 350 F F F 8 160-179 171 171 171 349 349 349 F F F 9 180-199 170 170 170 345 345 345 F F F 10 200-219 170 170 170 345 345 345 F F F 11 220-239 172 172 172 352 352 352 F F F 12 240-259 172 172 172 352 352 352 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 172 172 172 350 350 350 T T T 15 300-319 168 168 168 341 341 341 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 170 170 170 348 348 348 F F F 18 360-379 172 172 172 351 351 351 F F F 19 380-399 172 172 172 351 351 351 T T T 20 400-419 169 169 169 344 344 344 F F F 21 420-439 171 171 171 349 349 349 F F F 22 440-459 168 168 168 343 343 343 F F F 23 460-479 168 168 168 343 343 343 F F F 24 480-499 171 171 171 349 349 349 F F F 25 500-519 170 170 170 348 348 348 F F F 26 520-539 169 169 169 344 344 344 T T T 27 540-559 167 167 167 340 340 340 F F F 28 560-579 170 170 170 348 348 348 F F F 29 580-599 172 172 172 352 352 352 F F F Max gen: 172, Max flush: 352 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 169 169 169 344 344 344 F F F 3 060-079 173 173 173 353 353 353 F F F 4 080-099 173 173 173 353 353 353 F F F 5 100-119 168 168 168 341 341 341 F F F 6 120-139 170 170 170 346 346 346 F F F 7 140-159 172 172 172 350 350 350 F F F 8 160-179 171 171 171 349 349 349 F F F 9 180-199 170 170 170 345 345 345 F F F 10 200-219 170 170 170 345 345 345 F F F 11 220-239 172 172 172 352 352 352 F F F 12 240-259 172 172 172 352 352 352 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 173 173 173 353 353 353 F F F 15 300-319 168 168 168 341 341 341 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 170 170 170 348 348 348 F F F 18 360-379 172 172 172 351 351 351 F F F 19 380-399 173 173 173 353 353 353 F F F 20 400-419 169 169 169 344 344 344 F F F 21 420-439 171 171 171 349 349 349 F F F 22 440-459 168 168 168 343 343 343 F F F 23 460-479 168 168 168 343 343 343 F F F 24 480-499 171 171 171 349 349 349 F F F 25 500-519 170 170 170 348 348 348 F F F 26 520-539 173 173 173 353 353 353 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 170 170 170 348 348 348 F F F 29 580-599 172 172 172 352 352 352 F F F Max gen: 173, Max flush: 353 Begin loop 87 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.595995323Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.596256196Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.596279447Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878} {"msg":"Crucible cdf2c6b4-09ad-4de5-8af5-60734849fa95 has session id: f4395e25-9c18-49d1-a3ac-6023bea533a4","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.596729493Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Upstairs opts: Upstairs UUID: cdf2c6b4-09ad-4de5-8af5-60734849fa95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.596763943Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Crucible stats registered with UUID: cdf2c6b4-09ad-4de5-8af5-60734849fa95","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.596779873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597117598Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878} {"msg":"cdf2c6b4-09ad-4de5-8af5-60734849fa95 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597146538Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597160158Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597177258Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597201368Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597226789Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597249569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597272659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.59729763Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.59732166Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.59734486Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34501)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597521452Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38471)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597629394Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:26.597818756Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:26.597857537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:26.597951218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597972648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.597987058Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.5980924Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.598437884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.598464075Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.598478775Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.598495055Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.599110183Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.599138793Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.599738181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.599765391Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.599780181Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.599792541Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.599805912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:26.599819352Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.599047532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49280)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.599284685Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.600556741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.600581631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.60122551Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601827667Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601856707Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[0]R flush_numbers[0..12]: [336, 339, 344, 353, 353, 341, 346, 350, 349, 345, 345, 352]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601873218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[0]R generation[0..12]: [165, 166, 169, 173, 173, 168, 170, 172, 171, 170, 170, 172]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601888288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601902978Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[1]R flush_numbers[0..12]: [336, 339, 344, 353, 353, 341, 346, 350, 349, 345, 345, 352]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601922018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[1]R generation[0..12]: [165, 166, 169, 173, 173, 168, 170, 172, 171, 170, 170, 172]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601936488Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601951079Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[2]R flush_numbers[0..12]: [336, 339, 344, 353, 353, 341, 346, 350, 349, 345, 345, 352]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601965219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[2]R generation[0..12]: [165, 166, 169, 173, 173, 168, 170, 172, 171, 170, 170, 172]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601979429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.601993699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.60200682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.60202048Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 354","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.60203741Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.6020508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"":"downstairs","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.60206455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.60207689Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"0","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.602090221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"1","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.602103461Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"client":"2","":"downstairs","session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.602116901Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.602131391Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"cdf2c6b4-09ad-4de5-8af5-60734849fa95 is now active with session: f4395e25-9c18-49d1-a3ac-6023bea533a4","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.602144521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.602156581Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878,"session_id":"f4395e25-9c18-49d1-a3ac-6023bea533a4"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:36.602168822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 28672 data: 13 14 15 15 15 14 13 02/30 Read block 413 len 16384 03/30 Read block 382 len 32768 04/30 Read block 32 len 40960 05/30 Write block 126 len 36864 data: 11 11 12 13 20 19 16 16 18 06/30 Read block 130 len 8192 07/30 Write block 103 len 28672 data: 16 14 12 12 12 13 11 08/30 Write block 466 len 32768 data: 16 15 13 11 12 9 9 12 09/30 Read block 557 len 32768 10/30 Flush 11/30 Write block 194 len 20480 data: 14 13 16 17 15 12/30 Write block 353 len 32768 data: 15 15 17 18 14 15 12 11 13/30 Read block 540 len 12288 14/30 Write block 71 len 20480 data: 15 14 17 14 15 15/30 Write block 433 len 40960 data: 12 9 9 9 11 10 12 11 16 14 16/30 Read block 500 len 32768 17/30 Write block 142 len 20480 data: 11 18 18 16 17 18/30 Write block 208 len 40960 data: 15 16 15 14 14 12 11 11 11 9 19/30 Read block 60 len 16384 20/30 Read block 315 len 12288 21/30 Flush 22/30 Write block 380 len 36864 data: 19 16 16 12 9 10 16 13 15 23/30 Read block 156 len 16384 24/30 Read block 355 len 32768 25/30 Read block 249 len 4096 26/30 Write block 592 len 16384 data: 13 9 9 8 27/30 Read block 133 len 4096 28/30 Write block 379 len 16384 data: 22 20 17 17 29/30 Read block 275 len 32768 30/30 Read block 29 len 32768 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 9 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Read 8 Done Done Done false 31 Acked 1030 Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:36.632314784Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 169 169 169 344 344 344 F F F 3 060-079 174 174 174 355 355 355 F F F 4 080-099 173 173 173 353 353 353 F F F 5 100-119 174 174 174 354 354 354 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 174 174 174 355 355 355 F F F 8 160-179 171 171 171 349 349 349 F F F 9 180-199 174 174 174 355 355 355 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 172 172 172 352 352 352 F F F 12 240-259 172 172 172 352 352 352 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 173 173 173 353 353 353 F F F 15 300-319 168 168 168 341 341 341 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 174 174 174 355 355 355 T T T 19 380-399 173 173 173 353 353 353 T T T 20 400-419 169 169 169 344 344 344 F F F 21 420-439 174 174 174 355 355 355 F F F 22 440-459 174 174 174 355 355 355 F F F 23 460-479 174 174 174 354 354 354 F F F 24 480-499 174 174 174 354 354 354 F F F 25 500-519 174 174 174 354 354 354 F F F 26 520-539 173 173 173 353 353 353 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 170 170 170 348 348 348 F F F 29 580-599 172 172 172 352 352 352 T T T Max gen: 174, Max flush: 356 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 336 336 336 F F F 1 020-039 166 166 166 339 339 339 F F F 2 040-059 169 169 169 344 344 344 F F F 3 060-079 174 174 174 355 355 355 F F F 4 080-099 173 173 173 353 353 353 F F F 5 100-119 174 174 174 354 354 354 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 174 174 174 355 355 355 F F F 8 160-179 171 171 171 349 349 349 F F F 9 180-199 174 174 174 355 355 355 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 172 172 172 352 352 352 F F F 12 240-259 172 172 172 352 352 352 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 173 173 173 353 353 353 F F F 15 300-319 168 168 168 341 341 341 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 169 169 169 344 344 344 F F F 21 420-439 174 174 174 355 355 355 F F F 22 440-459 174 174 174 355 355 355 F F F 23 460-479 174 174 174 354 354 354 F F F 24 480-499 174 174 174 354 354 354 F F F 25 500-519 174 174 174 354 354 354 F F F 26 520-539 173 173 173 353 353 353 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 170 170 170 348 348 348 F F F 29 580-599 175 175 175 357 357 357 F F F Max gen: 175, Max flush: 357 Begin loop 88 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.836813179Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.837141293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.837174804Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886} {"msg":"Crucible 62a7a2ed-57df-48b2-b5a8-53f68f37d841 has session id: b78498d8-e0a0-45e5-b48d-2e507bae582c","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.837405856Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Upstairs opts: Upstairs UUID: 62a7a2ed-57df-48b2-b5a8-53f68f37d841, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.837496897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Crucible stats registered with UUID: 62a7a2ed-57df-48b2-b5a8-53f68f37d841","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.837526548Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.837977764Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886} {"msg":"62a7a2ed-57df-48b2-b5a8-53f68f37d841 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838003274Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838018494Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838033124Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838047124Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838060705Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838073945Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838087695Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838180756Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838209606Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838238017Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43472)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838536361Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35568)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838616382Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:50.838652332Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:50.838675132Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:01:50.838692653Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838705973Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838719443Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.838920186Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.839523553Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.839550534Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.839566124Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.839580354Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.840179822Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.840296293Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.840716428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.840739629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.840754139Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.84082571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.84084398Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:01:50.84085755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.840372654Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65324)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.840725218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.841947054Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.841975314Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.842459631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.842990377Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843013747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[0]R flush_numbers[0..12]: [336, 339, 344, 355, 353, 354, 354, 355, 349, 355, 356, 352]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843032228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[0]R generation[0..12]: [165, 166, 169, 174, 173, 174, 174, 174, 171, 174, 174, 172]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843050228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843074378Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[1]R flush_numbers[0..12]: [336, 339, 344, 355, 353, 354, 354, 355, 349, 355, 356, 352]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843092599Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[1]R generation[0..12]: [165, 166, 169, 174, 173, 174, 174, 174, 171, 174, 174, 172]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843110149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843127929Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[2]R flush_numbers[0..12]: [336, 339, 344, 355, 353, 354, 354, 355, 349, 355, 356, 352]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843155989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[2]R generation[0..12]: [165, 166, 169, 174, 173, 174, 174, 174, 171, 174, 174, 172]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-07-10T16:02:00.84318921Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.8432219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843246801Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843264361Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Next flush: 358","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843281661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843309231Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"":"downstairs","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843340072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843367132Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"0","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843391682Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"1","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843425833Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"client":"2","":"downstairs","session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843456393Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843476544Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"62a7a2ed-57df-48b2-b5a8-53f68f37d841 is now active with session: b78498d8-e0a0-45e5-b48d-2e507bae582c","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843505044Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843532304Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886,"session_id":"b78498d8-e0a0-45e5-b48d-2e507bae582c"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:00.843561294Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 8192 data: 13 11 02/30 Write block 291 len 16384 data: 11 12 14 14 03/30 Read block 281 len 4096 04/30 Read block 508 len 8192 05/30 Write block 35 len 28672 data: 7 7 6 7 9 9 12 06/30 Read block 214 len 40960 07/30 Read block 372 len 16384 08/30 Read block 544 len 16384 09/30 Read block 349 len 12288 10/30 Read block 155 len 36864 11/30 Write block 8 len 32768 data: 16 12 10 12 7 10 10 9 12/30 Flush 13/30 Write block 528 len 20480 data: 20 21 20 21 19 14/30 Read block 596 len 16384 15/30 Write block 435 len 20480 data: 10 10 12 11 13 16/30 Flush 17/30 Flush 18/30 Write block 227 len 4096 data: 15 19/30 Read block 155 len 24576 20/30 Read block 418 len 12288 21/30 Write block 529 len 40960 data: 22 21 22 20 12 13 11 13 14 17 22/30 Read block 175 len 28672 23/30 Write block 572 len 20480 data: 11 10 13 12 15 24/30 Write block 423 len 28672 data: 11 14 14 14 16 16 17 25/30 Flush 26/30 Write block 487 len 4096 data: 15 27/30 Read block 96 len 36864 28/30 Write block 551 len 40960 data: 17 15 13 12 11 8 12 10 7 7 29/30 Read block 74 len 40960 30/30 Read block 238 len 28672 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:02:00.87235799Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 358 358 358 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 176 176 176 358 358 358 F F F 3 060-079 174 174 174 355 355 355 F F F 4 080-099 173 173 173 353 353 353 F F F 5 100-119 174 174 174 354 354 354 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 174 174 174 355 355 355 F F F 8 160-179 171 171 171 349 349 349 F F F 9 180-199 174 174 174 355 355 355 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 176 176 176 361 361 361 F F F 12 240-259 172 172 172 352 352 352 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 176 176 176 358 358 358 F F F 15 300-319 168 168 168 341 341 341 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 169 169 169 344 344 344 F F F 21 420-439 176 176 176 361 361 361 F F F 22 440-459 174 174 174 355 355 355 F F F 23 460-479 174 174 174 354 354 354 F F F 24 480-499 174 174 174 354 354 354 T T T 25 500-519 174 174 174 354 354 354 F F F 26 520-539 176 176 176 361 361 361 F F F 27 540-559 167 167 167 340 340 340 T T T 28 560-579 176 176 176 361 361 361 T T T 29 580-599 175 175 175 357 357 357 F F F Max gen: 176, Max flush: 361 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 358 358 358 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 176 176 176 358 358 358 F F F 3 060-079 174 174 174 355 355 355 F F F 4 080-099 173 173 173 353 353 353 F F F 5 100-119 174 174 174 354 354 354 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 174 174 174 355 355 355 F F F 8 160-179 171 171 171 349 349 349 F F F 9 180-199 174 174 174 355 355 355 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 176 176 176 361 361 361 F F F 12 240-259 172 172 172 352 352 352 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 176 176 176 358 358 358 F F F 15 300-319 168 168 168 341 341 341 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 169 169 169 344 344 344 F F F 21 420-439 176 176 176 361 361 361 F F F 22 440-459 174 174 174 355 355 355 F F F 23 460-479 174 174 174 354 354 354 F F F 24 480-499 177 177 177 362 362 362 F F F 25 500-519 174 174 174 354 354 354 F F F 26 520-539 176 176 176 361 361 361 F F F 27 540-559 177 177 177 362 362 362 F F F 28 560-579 177 177 177 362 362 362 F F F 29 580-599 175 175 175 357 357 357 F F F Max gen: 177, Max flush: 362 Begin loop 89 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.073562843Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.07409852Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.074134911Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894} {"msg":"Crucible fc156642-df50-47fd-a919-e10b2c5cfc76 has session id: 69226aea-7e32-488d-82c9-5d58b9a1a958","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.074402104Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Upstairs opts: Upstairs UUID: fc156642-df50-47fd-a919-e10b2c5cfc76, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.074448975Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Crucible stats registered with UUID: fc156642-df50-47fd-a919-e10b2c5cfc76","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.074474205Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.074746488Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894} {"msg":"fc156642-df50-47fd-a919-e10b2c5cfc76 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.074829659Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.07484845Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.0748635Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.07487774Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.07489162Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.07490494Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.07491857Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.074976341Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.075016532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.075036732Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:02:15.075246535Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:02:15.075300485Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_connection connected from Ok(127.0.0.1:50066)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.075327196Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:02:15.075353376Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.075376196Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.075399377Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_connection connected from Ok(127.0.0.1:48779)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.075422797Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.075573629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.076105456Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.076132926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.076149416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.076163476Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.076734624Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.076862005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.07723657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.07727301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.077299261Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.077377542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.077397752Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:15.077411662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.076771694Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_connection connected from Ok(127.0.0.1:58655)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.076939086Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.07807421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.078103101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.078614787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079140954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079159444Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[0]R flush_numbers[0..12]: [358, 358, 358, 355, 353, 354, 354, 355, 349, 355, 356, 361]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079180875Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[0]R generation[0..12]: [176, 176, 176, 174, 173, 174, 174, 174, 171, 174, 174, 176]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079195404Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079215275Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[1]R flush_numbers[0..12]: [358, 358, 358, 355, 353, 354, 354, 355, 349, 355, 356, 361]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079229965Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[1]R generation[0..12]: [176, 176, 176, 174, 173, 174, 174, 174, 171, 174, 174, 176]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079244185Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079258455Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[2]R flush_numbers[0..12]: [358, 358, 358, 355, 353, 354, 354, 355, 349, 355, 356, 361]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079272505Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[2]R generation[0..12]: [176, 176, 176, 174, 173, 174, 174, 174, 171, 174, 174, 176]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079286376Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079300466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079313746Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079332256Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Next flush: 363","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079346126Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079359457Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"":"downstairs","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079372967Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079385287Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"0","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079399617Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"1","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079412897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"client":"2","":"downstairs","session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079429058Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079441588Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"fc156642-df50-47fd-a919-e10b2c5cfc76 is now active with session: 69226aea-7e32-488d-82c9-5d58b9a1a958","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079454648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079466728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894,"session_id":"69226aea-7e32-488d-82c9-5d58b9a1a958"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:25.079479098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 547 len 12288 data: 9 12 15 03/30 Write block 309 len 28672 data: 11 10 6 6 7 10 8 04/30 Write block 552 len 36864 data: 16 14 13 12 9 13 11 8 8 05/30 Read block 84 len 40960 06/30 Write block 173 len 24576 data: 13 14 10 10 11 10 07/30 Flush 08/30 Write block 237 len 40960 data: 15 13 14 13 16 16 17 17 15 19 09/30 Read block 520 len 24576 10/30 Write block 549 len 24576 data: 16 17 18 17 15 14 11/30 Read block 224 len 4096 12/30 Read block 48 len 36864 13/30 Read block 234 len 4096 14/30 Write block 512 len 4096 data: 8 15/30 Write block 473 len 32768 data: 13 12 16 19 17 14 14 13 16/30 Read block 105 len 8192 17/30 Read block 220 len 24576 18/30 Read block 37 len 28672 19/30 Read block 66 len 16384 20/30 Write block 484 len 4096 data: 15 21/30 Flush 22/30 Write block 74 len 4096 data: 15 23/30 Write block 432 len 20480 data: 15 13 10 11 11 24/30 Read block 51 len 24576 25/30 Write block 95 len 12288 data: 10 10 10 26/30 Read block 546 len 32768 27/30 Read block 304 len 16384 28/30 Write block 188 len 4096 data: 10 29/30 Read block 8 len 40960 30/30 Read block 312 len 8192 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false 31 Acked 1030 Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 9 8 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:02:25.107140429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 358 358 358 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 176 176 176 358 358 358 F F F 3 060-079 174 174 174 355 355 355 T T T 4 080-099 173 173 173 353 353 353 T T T 5 100-119 174 174 174 354 354 354 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 174 174 174 355 355 355 F F F 8 160-179 178 178 178 364 364 364 F F F 9 180-199 174 174 174 355 355 355 T T T 10 200-219 174 174 174 356 356 356 F F F 11 220-239 178 178 178 365 365 365 F F F 12 240-259 178 178 178 365 365 365 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 176 176 176 358 358 358 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 169 169 169 344 344 344 F F F 21 420-439 176 176 176 361 361 361 T T T 22 440-459 174 174 174 355 355 355 F F F 23 460-479 178 178 178 365 365 365 F F F 24 480-499 178 178 178 366 366 366 F F F 25 500-519 178 178 178 365 365 365 F F F 26 520-539 176 176 176 361 361 361 F F F 27 540-559 178 178 178 365 365 365 F F F 28 560-579 178 178 178 364 364 364 F F F 29 580-599 175 175 175 357 357 357 F F F Max gen: 178, Max flush: 366 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 358 358 358 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 176 176 176 358 358 358 F F F 3 060-079 179 179 179 367 367 367 F F F 4 080-099 179 179 179 367 367 367 F F F 5 100-119 174 174 174 354 354 354 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 174 174 174 355 355 355 F F F 8 160-179 178 178 178 364 364 364 F F F 9 180-199 179 179 179 367 367 367 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 178 178 178 365 365 365 F F F 12 240-259 178 178 178 365 365 365 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 176 176 176 358 358 358 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 169 169 169 344 344 344 F F F 21 420-439 179 179 179 367 367 367 F F F 22 440-459 174 174 174 355 355 355 F F F 23 460-479 178 178 178 365 365 365 F F F 24 480-499 178 178 178 366 366 366 F F F 25 500-519 178 178 178 365 365 365 F F F 26 520-539 176 176 176 361 361 361 F F F 27 540-559 178 178 178 365 365 365 F F F 28 560-579 178 178 178 364 364 364 F F F 29 580-599 175 175 175 357 357 357 F F F Max gen: 179, Max flush: 367 Begin loop 90 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.320654599Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.320970053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.320993453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902} {"msg":"Crucible 8b53fa48-4d4e-454e-8b9b-5427fcb2d12e has session id: a5543764-8742-413a-8e25-31d2f45bda40","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321303537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Upstairs opts: Upstairs UUID: 8b53fa48-4d4e-454e-8b9b-5427fcb2d12e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321347948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Crucible stats registered with UUID: 8b53fa48-4d4e-454e-8b9b-5427fcb2d12e","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321372618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321656312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902} {"msg":"8b53fa48-4d4e-454e-8b9b-5427fcb2d12e active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321691332Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321706892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321721522Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321735233Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321749113Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321762803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321776333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321829154Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321850624Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.321868784Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:02:39.322044247Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_connection connected from Ok(127.0.0.1:45526)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.322081737Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:02:39.322109618Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:02:39.322135318Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_connection connected from Ok(127.0.0.1:46079)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.322155248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.322178828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.322199829Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.322427282Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.322894397Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.322929948Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.322957288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.322983518Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.323546076Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.323655577Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.324032382Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.324067722Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.324092793Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.324192794Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.324220074Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:39.324234974Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.323646067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_connection connected from Ok(127.0.0.1:59654)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.324048782Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.324926273Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.324954094Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.325514281Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326129849Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326150669Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[0]R flush_numbers[0..12]: [358, 358, 358, 367, 367, 354, 354, 355, 364, 367, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326165639Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[0]R generation[0..12]: [176, 176, 176, 179, 179, 174, 174, 174, 178, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326179999Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326198829Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[1]R flush_numbers[0..12]: [358, 358, 358, 367, 367, 354, 354, 355, 364, 367, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326213179Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[1]R generation[0..12]: [176, 176, 176, 179, 179, 174, 174, 174, 178, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.32622727Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.32624195Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[2]R flush_numbers[0..12]: [358, 358, 358, 367, 367, 354, 354, 355, 364, 367, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.32625609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[2]R generation[0..12]: [176, 176, 176, 179, 179, 174, 174, 174, 178, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.32627018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326284491Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326297501Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326310901Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Next flush: 368","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326324071Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326341641Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"":"downstairs","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326355631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326367681Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"0","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326381092Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"1","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326396572Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"client":"2","":"downstairs","session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326410542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326422982Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"8b53fa48-4d4e-454e-8b9b-5427fcb2d12e is now active with session: a5543764-8742-413a-8e25-31d2f45bda40","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326435892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326447933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902,"session_id":"a5543764-8742-413a-8e25-31d2f45bda40"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:02:49.326460053Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 415 len 40960 data: 5 6 6 10 9 9 8 11 12 15 02/30 Read block 383 len 8192 03/30 Write block 529 len 4096 data: 23 04/30 Write block 65 len 8192 data: 17 15 05/30 Read block 522 len 12288 06/30 Read block 558 len 16384 07/30 Read block 283 len 4096 08/30 Read block 192 len 24576 09/30 Write block 569 len 20480 data: 5 8 8 12 11 10/30 Write block 426 len 36864 data: 15 17 17 18 16 14 16 14 11 11/30 Write block 252 len 8192 data: 16 14 12/30 Write block 487 len 20480 data: 16 14 13 13 15 13/30 Read block 158 len 28672 14/30 Read block 528 len 4096 15/30 Read block 384 len 20480 16/30 Write block 471 len 28672 data: 10 10 14 13 17 20 18 17/30 Flush 18/30 Write block 422 len 8192 data: 12 13 19/30 Write block 290 len 28672 data: 11 12 13 15 15 11 12 20/30 Read block 266 len 20480 21/30 Read block 72 len 32768 22/30 Read block 261 len 4096 23/30 Write block 538 len 32768 data: 18 13 12 9 11 11 14 10 24/30 Write block 246 len 8192 data: 20 19 25/30 Write block 466 len 8192 data: 17 16 26/30 Read block 469 len 20480 27/30 Write block 535 len 16384 data: 12 14 15 19 28/30 Read block 430 len 16384 29/30 Write block 438 len 40960 data: 12 14 12 17 15 14 15 16 16 19 30/30 Read block 251 len 28672 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Read 8 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 10 Done Done Done false 31 Acked 1030 Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:02:49.350328005Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 358 358 358 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 176 176 176 358 358 358 F F F 3 060-079 180 180 180 368 368 368 F F F 4 080-099 179 179 179 367 367 367 F F F 5 100-119 174 174 174 354 354 354 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 174 174 174 355 355 355 F F F 8 160-179 178 178 178 364 364 364 F F F 9 180-199 179 179 179 367 367 367 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 178 178 178 365 365 365 F F F 12 240-259 180 180 180 368 368 368 T T T 13 260-279 166 166 166 337 337 337 F F F 14 280-299 180 180 180 369 369 369 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 180 180 180 368 368 368 F F F 21 420-439 180 180 180 369 369 369 T T T 22 440-459 174 174 174 355 355 355 T T T 23 460-479 180 180 180 368 368 368 T T T 24 480-499 180 180 180 368 368 368 F F F 25 500-519 178 178 178 365 365 365 F F F 26 520-539 180 180 180 368 368 368 T T T 27 540-559 178 178 178 365 365 365 T T T 28 560-579 180 180 180 368 368 368 F F F 29 580-599 175 175 175 357 357 357 F F F Max gen: 180, Max flush: 369 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 358 358 358 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 176 176 176 358 358 358 F F F 3 060-079 180 180 180 368 368 368 F F F 4 080-099 179 179 179 367 367 367 F F F 5 100-119 174 174 174 354 354 354 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 174 174 174 355 355 355 F F F 8 160-179 178 178 178 364 364 364 F F F 9 180-199 179 179 179 367 367 367 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 178 178 178 365 365 365 F F F 12 240-259 181 181 181 370 370 370 F F F 13 260-279 166 166 166 337 337 337 F F F 14 280-299 180 180 180 369 369 369 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 170 170 170 345 345 345 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 180 180 180 368 368 368 F F F 21 420-439 181 181 181 370 370 370 F F F 22 440-459 181 181 181 370 370 370 F F F 23 460-479 181 181 181 370 370 370 F F F 24 480-499 180 180 180 368 368 368 F F F 25 500-519 178 178 178 365 365 365 F F F 26 520-539 181 181 181 370 370 370 F F F 27 540-559 181 181 181 370 370 370 F F F 28 560-579 180 180 180 368 368 368 F F F 29 580-599 175 175 175 357 357 357 F F F Max gen: 181, Max flush: 370 Begin loop 91 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.595963003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.596228096Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.596246566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910} {"msg":"Crucible 15db72b7-04af-41b9-ab07-32b67953d829 has session id: 0e82ace1-9f82-421a-a74a-605d4f3ac8d2","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.596681042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Upstairs opts: Upstairs UUID: 15db72b7-04af-41b9-ab07-32b67953d829, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.596716152Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Crucible stats registered with UUID: 15db72b7-04af-41b9-ab07-32b67953d829","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.596732043Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597141018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910} {"msg":"15db72b7-04af-41b9-ab07-32b67953d829 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597163988Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597177548Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597197278Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597221869Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597250039Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.59728186Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.59731349Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597348Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597378121Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597408421Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37968)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597452782Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:45693)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597486112Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:03.597517653Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:03.597573003Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:03.597673755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597709755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597737695Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.597783896Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.598366883Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.598393924Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.598409114Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.598422884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.599086173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.599217884Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.59968965Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.599713301Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.599727641Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.599783001Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.599801451Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:03.599815022Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.598377023Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34037)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.598509625Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.599905903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.599937013Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.600541961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601087838Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601116678Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[0]R flush_numbers[0..12]: [358, 358, 358, 368, 367, 354, 354, 355, 364, 367, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601141898Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[0]R generation[0..12]: [176, 176, 176, 180, 179, 174, 174, 174, 178, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601160428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601178569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[1]R flush_numbers[0..12]: [358, 358, 358, 368, 367, 354, 354, 355, 364, 367, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601196389Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[1]R generation[0..12]: [176, 176, 176, 180, 179, 174, 174, 174, 178, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601213819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.60123168Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[2]R flush_numbers[0..12]: [358, 358, 358, 368, 367, 354, 354, 355, 364, 367, 356, 365]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2024-07-10T16:03:13.60124958Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[2]R generation[0..12]: [176, 176, 176, 180, 179, 174, 174, 174, 178, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.60127128Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.60128928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.60130551Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.60132219Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Next flush: 371","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601346101Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601371921Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"":"downstairs","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601396371Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601423242Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"0","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601443002Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"1","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601465912Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"client":"2","":"downstairs","session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-07-10T16:03:13.601486333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601507803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"15db72b7-04af-41b9-ab07-32b67953d829 is now active with session: 0e82ace1-9f82-421a-a74a-605d4f3ac8d2","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601531083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601547933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910,"session_id":"0e82ace1-9f82-421a-a74a-605d4f3ac8d2"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:13.601560674Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 527 len 8192 data: 19 21 02/30 Write block 73 len 40960 data: 18 16 16 16 16 14 12 10 9 10 03/30 Read block 332 len 12288 04/30 Write block 114 len 12288 data: 12 10 12 05/30 Read block 112 len 16384 06/30 Write block 248 len 16384 data: 20 19 19 19 07/30 Flush 08/30 Write block 265 len 40960 data: 11 11 8 4 5 7 6 9 10 10 09/30 Read block 570 len 12288 10/30 Write block 275 len 40960 data: 10 8 8 11 10 9 9 11 10 8 11/30 Read block 180 len 4096 12/30 Read block 400 len 4096 13/30 Write block 434 len 24576 data: 12 12 12 13 13 15 14/30 Read block 165 len 32768 15/30 Flush 16/30 Write block 47 len 32768 data: 12 12 12 11 12 15 11 10 17/30 Read block 427 len 12288 18/30 Read block 64 len 24576 19/30 Flush 20/30 Flush 21/30 Flush 22/30 Write block 66 len 4096 data: 16 23/30 Write block 257 len 36864 data: 16 12 14 15 12 12 12 12 12 24/30 Read block 584 len 40960 25/30 Read block 337 len 20480 26/30 Write block 151 len 20480 data: 12 9 9 10 11 27/30 Read block 266 len 24576 28/30 Write block 329 len 4096 data: 13 29/30 Read block 151 len 12288 30/30 Write block 537 len 36864 data: 16 20 14 13 10 12 12 15 11 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false 31 Acked 1030 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:13.629296226Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 358 358 358 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 182 182 182 373 373 373 F F F 3 060-079 182 182 182 371 371 371 T T T 4 080-099 182 182 182 371 371 371 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 174 174 174 355 355 355 T T T 8 160-179 178 178 178 364 364 364 F F F 9 180-199 179 179 179 367 367 367 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 178 178 178 365 365 365 F F F 12 240-259 182 182 182 371 371 371 T T T 13 260-279 182 182 182 372 372 372 T T T 14 280-299 182 182 182 372 372 372 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 170 170 170 345 345 345 T T T 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 180 180 180 368 368 368 F F F 21 420-439 182 182 182 372 372 372 F F F 22 440-459 181 181 181 370 370 370 F F F 23 460-479 181 181 181 370 370 370 F F F 24 480-499 180 180 180 368 368 368 F F F 25 500-519 178 178 178 365 365 365 F F F 26 520-539 182 182 182 371 371 371 T T T 27 540-559 181 181 181 370 370 370 T T T 28 560-579 180 180 180 368 368 368 F F F 29 580-599 175 175 175 357 357 357 F F F Max gen: 182, Max flush: 373 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 358 358 358 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 182 182 182 373 373 373 F F F 3 060-079 183 183 183 374 374 374 F F F 4 080-099 182 182 182 371 371 371 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 183 183 183 374 374 374 F F F 8 160-179 178 178 178 364 364 364 F F F 9 180-199 179 179 179 367 367 367 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 178 178 178 365 365 365 F F F 12 240-259 183 183 183 374 374 374 F F F 13 260-279 183 183 183 374 374 374 F F F 14 280-299 182 182 182 372 372 372 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 183 183 183 374 374 374 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 180 180 180 368 368 368 F F F 21 420-439 182 182 182 372 372 372 F F F 22 440-459 181 181 181 370 370 370 F F F 23 460-479 181 181 181 370 370 370 F F F 24 480-499 180 180 180 368 368 368 F F F 25 500-519 178 178 178 365 365 365 F F F 26 520-539 183 183 183 374 374 374 F F F 27 540-559 183 183 183 374 374 374 F F F 28 560-579 180 180 180 368 368 368 F F F 29 580-599 175 175 175 357 357 357 F F F Max gen: 183, Max flush: 374 Begin loop 92 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.887459312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.887763216Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.887796866Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918} {"msg":"Crucible 1e5def87-52ee-4a33-b870-706a0b02b4fd has session id: 7a7f2004-53bd-4eb0-b5f9-c5163debffd2","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888309512Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Upstairs opts: Upstairs UUID: 1e5def87-52ee-4a33-b870-706a0b02b4fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888349563Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Crucible stats registered with UUID: 1e5def87-52ee-4a33-b870-706a0b02b4fd","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888365793Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888661197Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918} {"msg":"1e5def87-52ee-4a33-b870-706a0b02b4fd active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888756798Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888782969Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888799639Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888813719Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888827569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888841129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.888858799Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.88887237Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.88888661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.88890124Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43396)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.889098582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34821)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.889287305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:27.889328605Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:27.889359096Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:27.889386456Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.889403576Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.889417697Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.889541628Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.889888903Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.889916693Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.889932913Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.889950613Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.890425839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.890538211Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.890978966Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.891015697Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.891030847Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.891043357Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.891056957Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:27.891070387Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.889868262Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:53747)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.889959843Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.891032297Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.891080148Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.891708816Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892310083Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892348594Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[0]R flush_numbers[0..12]: [358, 358, 373, 374, 371, 371, 354, 374, 364, 367, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892369514Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[0]R generation[0..12]: [176, 176, 182, 183, 182, 182, 174, 183, 178, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892388124Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892406315Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[1]R flush_numbers[0..12]: [358, 358, 373, 374, 371, 371, 354, 374, 364, 367, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892424235Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[1]R generation[0..12]: [176, 176, 182, 183, 182, 182, 174, 183, 178, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892441545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892459415Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {Wait for a query_work_queue command to finish before sending IO "msg":"[2]R flush_numbers[0..12]: [358, 358, 373, 374, 371, 371, 354, 374, 364, 367, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892480765Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[2]R generation[0..12]: [176, 176, 182, 183, 182, 182, 174, 183, 178, 179, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892498366Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892516096Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892532436Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892549076Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Next flush: 375","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892565657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892582167Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"":"downstairs","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892601877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892617357Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"0","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892634257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"1","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892651287Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"client":"2","":"downstairs","session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892668598Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892684018Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"1e5def87-52ee-4a33-b870-706a0b02b4fd is now active with session: 7a7f2004-53bd-4eb0-b5f9-c5163debffd2","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892700208Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892715218Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918,"session_id":"7a7f2004-53bd-4eb0-b5f9-c5163debffd2"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:37.892730628Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 323 len 4096 data: 17 02/30 Read block 349 len 40960 03/30 Write block 11 len 24576 data: 13 8 11 11 10 13 04/30 Write block 577 len 32768 data: 14 17 14 12 8 8 8 6 05/30 Write block 511 len 4096 data: 10 06/30 Write block 513 len 12288 data: 8 8 8 07/30 Read block 19 len 40960 08/30 Read block 242 len 12288 09/30 Read block 431 len 4096 10/30 Read block 473 len 20480 11/30 Read block 293 len 40960 12/30 Read block 530 len 8192 13/30 Read block 435 len 20480 14/30 Write block 12 len 4096 data: 9 15/30 Flush 16/30 Write block 404 len 20480 data: 14 12 11 10 10 17/30 Read block 186 len 40960 18/30 Read block 465 len 36864 19/30 Read block 325 len 40960 20/30 Read block 458 len 32768 21/30 Read block 5 len 28672 22/30 Write block 403 len 8192 data: 15 15 23/30 Read block 24 len 8192 24/30 Read block 112 len 4096 25/30 Flush 26/30 Write block 50 len 32768 data: 12 13 16 12 11 10 14 12 27/30 Write block 454 len 32768 data: 11 10 10 7 7 7 9 11 28/30 Read block 2 len 4096 29/30 Read block 148 len 40960 30/30 Write block 187 len 20480 data: 9 11 13 12 16 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false 31 Acked 1030 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:37.919820712Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 375 375 375 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 182 182 182 373 373 373 T T T 3 060-079 183 183 183 374 374 374 F F F 4 080-099 182 182 182 371 371 371 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 183 183 183 374 374 374 F F F 8 160-179 178 178 178 364 364 364 F F F 9 180-199 179 179 179 367 367 367 T T T 10 200-219 174 174 174 356 356 356 F F F 11 220-239 178 178 178 365 365 365 F F F 12 240-259 183 183 183 374 374 374 F F F 13 260-279 183 183 183 374 374 374 F F F 14 280-299 182 182 182 372 372 372 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 184 184 184 375 375 375 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 184 184 184 377 377 377 F F F 21 420-439 182 182 182 372 372 372 F F F 22 440-459 181 181 181 370 370 370 T T T 23 460-479 181 181 181 370 370 370 T T T 24 480-499 180 180 180 368 368 368 F F F 25 500-519 184 184 184 375 375 375 F F F 26 520-539 183 183 183 374 374 374 F F F 27 540-559 183 183 183 374 374 374 F F F 28 560-579 184 184 184 375 375 375 F F F 29 580-599 184 184 184 375 375 375 F F F Max gen: 184, Max flush: 377 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 375 375 375 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 185 185 185 378 378 378 F F F 3 060-079 183 183 183 374 374 374 F F F 4 080-099 182 182 182 371 371 371 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 183 183 183 374 374 374 F F F 8 160-179 178 178 178 364 364 364 F F F 9 180-199 185 185 185 378 378 378 F F F 10 200-219 174 174 174 356 356 356 F F F 11 220-239 178 178 178 365 365 365 F F F 12 240-259 183 183 183 374 374 374 F F F 13 260-279 183 183 183 374 374 374 F F F 14 280-299 182 182 182 372 372 372 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 184 184 184 375 375 375 F F F 17 340-359 174 174 174 355 355 355 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 175 175 175 357 357 357 F F F 20 400-419 184 184 184 377 377 377 F F F 21 420-439 182 182 182 372 372 372 F F F 22 440-459 185 185 185 378 378 378 F F F 23 460-479 185 185 185 378 378 378 F F F 24 480-499 180 180 180 368 368 368 F F F 25 500-519 184 184 184 375 375 375 F F F 26 520-539 183 183 183 374 374 374 F F F 27 540-559 183 183 183 374 374 374 F F F 28 560-579 184 184 184 375 375 375 F F F 29 580-599 184 184 184 375 375 375 F F F Max gen: 185, Max flush: 378 Begin loop 93 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.140577513Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.140867247Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.140895057Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926} {"msg":"Crucible c0b9a104-6b3d-4fb9-a8cc-fccb736861c8 has session id: 452f71a2-1bc2-4c63-9cdf-b0502a55e717","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.141264672Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Upstairs opts: Upstairs UUID: c0b9a104-6b3d-4fb9-a8cc-fccb736861c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.141316313Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Crucible stats registered with UUID: c0b9a104-6b3d-4fb9-a8cc-fccb736861c8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.141342453Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.14184867Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926} {"msg":"c0b9a104-6b3d-4fb9-a8cc-fccb736861c8 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.14187184Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.14188678Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.14190168Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.14191559Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.141929221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.141942721Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.141956221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.142046152Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.142067192Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.142082603Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:52.142260395Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_connection connected from Ok(127.0.0.1:52859)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.142424927Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:52.142464108Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:03:52.142495448Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.142518748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.142543609Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_connection connected from Ok(127.0.0.1:34901)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.142569169Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.142594199Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.142975874Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.143001424Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.143128376Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.143155386Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.14346795Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.143594972Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.143929436Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.143952216Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.143975977Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.144165329Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.144193079Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:03:52.14420901Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.143967127Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_connection connected from Ok(127.0.0.1:51411)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.144342481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.145290173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.145316954Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.145872631Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146468508Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146497459Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[0]R flush_numbers[0..12]: [375, 358, 378, 374, 371, 371, 354, 374, 364, 378, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146521389Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[0]R generation[0..12]: [184, 176, 185, 183, 182, 182, 174, 183, 178, 185, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.14654022Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.14655827Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[1]R flush_numbers[0..12]: [375, 358, 378, 374, 371, 371, 354, 374, 364, 378, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.14657655Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[1]R generation[0..12]: [184, 176, 185, 183, 182, 182, 174, 183, 178, 185, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.14659433Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.14661216Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[2]R flush_numbers[0..12]: [375, 358, 378, 374, 371, 371, 354, 374, 364, 378, 356, 365]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146629661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[2]R generation[0..12]: [184, 176, 185, 183, 182, 182, 174, 183, 178, 185, 174, 178]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146651971Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146670141Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146686711Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146703402Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Next flush: 379","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146719972Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146736392Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"":"downstairs","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146753432Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146770742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"0","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146788032Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"1","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146804963Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"client":"2","":"downstairs","session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146822073Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146837433Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"c0b9a104-6b3d-4fb9-a8cc-fccb736861c8 is now active with session: 452f71a2-1bc2-4c63-9cdf-b0502a55e717","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146853923Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146868913Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926,"session_id":"452f71a2-1bc2-4c63-9cdf-b0502a55e717"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:02.146884284Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 177 len 20480 data: 12 11 11 9 7 02/30 Flush 03/30 Write block 350 len 32768 data: 12 12 14 16 16 18 19 15 04/30 Read block 476 len 12288 05/30 Write block 477 len 24576 data: 19 15 15 14 13 12 06/30 Read block 136 len 24576 07/30 Read block 508 len 36864 08/30 Write block 321 len 16384 data: 13 17 18 17 09/30 Read block 414 len 16384 10/30 Read block 280 len 28672 11/30 Flush 12/30 Write block 398 len 12288 data: 19 19 18 13/30 Write block 498 len 32768 data: 16 16 16 15 14 13 13 11 14/30 Read block 73 len 32768 15/30 Write block 216 len 40960 data: 12 10 10 12 13 18 18 14 16 14 16/30 Read block 59 len 36864 17/30 Write block 344 len 40960 data: 15 14 14 16 17 13 13 13 15 17 18/30 Write block 533 len 24576 data: 13 14 13 15 17 21 19/30 Read block 417 len 28672 20/30 Write block 425 len 12288 data: 15 16 18 21/30 Flush 22/30 Write block 433 len 36864 data: 15 13 13 13 14 14 16 13 18 23/30 Write block 157 len 28672 data: 12 15 14 10 11 10 8 24/30 Read block 50 len 4096 25/30 Read block 596 len 16384 26/30 Flush 27/30 Write block 12 len 24576 data: 10 12 12 11 14 12 28/30 Write block 79 len 4096 data: 13 29/30 Write block 444 len 36864 data: 16 17 17 20 16 14 12 16 17 30/30 Read block 492 len 16384 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Write 9 Done Done Sent false 31 Acked 1030 Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 2 3 Done 3 4 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:04:02.179426757Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 375 375 375 T T T 1 020-039 176 176 176 358 358 358 F F F 2 040-059 185 185 185 378 378 378 F F F 3 060-079 183 183 183 374 374 374 T T T 4 080-099 182 182 182 371 371 371 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 186 186 186 383 383 383 F F F 8 160-179 186 186 186 383 383 383 F F F 9 180-199 186 186 186 379 379 379 F F F 10 200-219 186 186 186 381 381 381 F F F 11 220-239 186 186 186 381 381 381 F F F 12 240-259 183 183 183 374 374 374 F F F 13 260-279 183 183 183 374 374 374 F F F 14 280-299 182 182 182 372 372 372 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 186 186 186 380 380 380 F F F 17 340-359 186 186 186 381 381 381 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 186 186 186 381 381 381 F F F 20 400-419 186 186 186 381 381 381 F F F 21 420-439 186 186 186 383 383 383 F F F 22 440-459 186 186 186 383 383 383 T T T 23 460-479 186 186 186 380 380 380 F F F 24 480-499 186 186 186 381 381 381 F F F 25 500-519 186 186 186 381 381 381 F F F 26 520-539 186 186 186 382 382 382 F F F 27 540-559 183 183 183 374 374 374 F F F 28 560-579 184 184 184 375 375 375 F F F 29 580-599 184 184 184 375 375 375 F F F Max gen: 186, Max flush: 383 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 176 176 176 358 358 358 F F F 2 040-059 185 185 185 378 378 378 F F F 3 060-079 187 187 187 384 384 384 F F F 4 080-099 182 182 182 371 371 371 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 174 174 174 354 354 354 F F F 7 140-159 186 186 186 383 383 383 F F F 8 160-179 186 186 186 383 383 383 F F F 9 180-199 186 186 186 379 379 379 F F F 10 200-219 186 186 186 381 381 381 F F F 11 220-239 186 186 186 381 381 381 F F F 12 240-259 183 183 183 374 374 374 F F F 13 260-279 183 183 183 374 374 374 F F F 14 280-299 182 182 182 372 372 372 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 186 186 186 380 380 380 F F F 17 340-359 186 186 186 381 381 381 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 186 186 186 381 381 381 F F F 20 400-419 186 186 186 381 381 381 F F F 21 420-439 186 186 186 383 383 383 F F F 22 440-459 187 187 187 384 384 384 F F F 23 460-479 186 186 186 380 380 380 F F F 24 480-499 186 186 186 381 381 381 F F F 25 500-519 186 186 186 381 381 381 F F F 26 520-539 186 186 186 382 382 382 F F F 27 540-559 183 183 183 374 374 374 F F F 28 560-579 184 184 184 375 375 375 F F F 29 580-599 184 184 184 375 375 375 F F F Max gen: 187, Max flush: 384 Begin loop 94 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.384024833Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.384389958Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.384417928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934} {"msg":"Crucible 2b49f7b7-c0ac-42af-81cf-5ca91d37ce87 has session id: 9cde4083-7be8-44d8-be0a-720a469eb891","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.384666511Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Upstairs opts: Upstairs UUID: 2b49f7b7-c0ac-42af-81cf-5ca91d37ce87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.384710742Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Crucible stats registered with UUID: 2b49f7b7-c0ac-42af-81cf-5ca91d37ce87","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.384734362Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385085116Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934} {"msg":"2b49f7b7-c0ac-42af-81cf-5ca91d37ce87 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385125317Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385150427Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385176298Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385198588Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385221028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385244238Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385268509Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385298779Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385324319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.38538437Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_connection connected from Ok(127.0.0.1:63560)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385573333Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_connection connected from Ok(127.0.0.1:40216)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385715335Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:04:16.385741565Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:04:16.385758795Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:04:16.385775305Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385788245Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385801585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.385917827Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.386360373Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.386391873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.386417713Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.386450734Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.386992171Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.387128372Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.387471067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.387513787Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.387529938Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.387601518Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.387619989Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:16.387633569Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.387062321Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_connection connected from Ok(127.0.0.1:62258)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.387345045Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.388623151Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.388653582Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389166928Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389708145Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389732206Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[0]R flush_numbers[0..12]: [384, 358, 378, 384, 371, 371, 354, 383, 383, 379, 381, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389752336Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[0]R generation[0..12]: [187, 176, 185, 187, 182, 182, 174, 186, 186, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389767276Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389782026Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[1]R flush_numbers[0..12]: [384, 358, 378, 384, 371, 371, 354, 383, 383, 379, 381, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389796416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[1]R generation[0..12]: [187, 176, 185, 187, 182, 182, 174, 186, 186, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389810687Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389825037Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[2]R flush_numbers[0..12]: [384, 358, 378, 384, 371, 371, 354, 383, 383, 379, 381, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389839067Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[2]R generation[0..12]: [187, 176, 185, 187, 182, 182, 174, 186, 186, 186, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389852927Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389867277Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389884197Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389897768Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Next flush: 385","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389910908Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389924098Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"":"downstairs","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389940118Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389952518Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"0","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389966378Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"1","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389979938Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"client":"2","":"downstairs","session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.389993429Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.390005839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"2b49f7b7-c0ac-42af-81cf-5ca91d37ce87 is now active with session: 9cde4083-7be8-44d8-be0a-720a469eb891","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.390018799Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.390030829Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934,"session_id":"9cde4083-7be8-44d8-be0a-720a469eb891"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:26.390043109Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 200 len 12288 data: 14 12 12 02/30 Flush 03/30 Flush 04/30 Write block 129 len 8192 data: 14 21 05/30 Write block 33 len 36864 data: 7 7 8 8 7 8 10 10 13 06/30 Read block 402 len 8192 07/30 Write block 210 len 20480 data: 16 15 15 13 12 08/30 Read block 536 len 32768 09/30 Read block 178 len 36864 10/30 Write block 523 len 28672 data: 9 12 13 19 20 22 24 11/30 Write block 147 len 8192 data: 20 18 12/30 Read block 313 len 40960 13/30 Read block 147 len 40960 14/30 Write block 384 len 40960 data: 10 11 17 14 16 18 20 19 17 19 15/30 Read block 580 len 8192 16/30 Write block 191 len 40960 data: 17 17 15 15 14 17 18 16 14 15 17/30 Read block 178 len 8192 18/30 Read block 271 len 20480 19/30 Write block 442 len 12288 data: 16 15 17 20/30 Read block 429 len 36864 21/30 Write block 251 len 32768 data: 20 17 15 19 21 19 17 13 22/30 Read block 339 len 4096 23/30 Write block 48 len 36864 data: 13 13 13 14 17 13 12 11 15 24/30 Read block 365 len 16384 25/30 Write block 510 len 8192 data: 10 11 26/30 Read block 484 len 12288 27/30 Read block 355 len 28672 28/30 Write block 385 len 24576 data: 12 18 15 17 19 21 29/30 Read block 236 len 28672 30/30 Write block 523 len 20480 data: 10 13 14 20 21 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Write 6 Done Done Done false 30 Acked 1029 Read 7 Done Done Done false 31 Acked 1030 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:04:26.415925728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 188 188 188 387 387 387 F F F 2 040-059 188 188 188 387 387 387 T T T 3 060-079 187 187 187 384 384 384 F F F 4 080-099 182 182 182 371 371 371 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 188 188 188 387 387 387 F F F 8 160-179 186 186 186 383 383 383 F F F 9 180-199 188 188 188 387 387 387 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 186 186 186 381 381 381 F F F 12 240-259 183 183 183 374 374 374 T T T 13 260-279 183 183 183 374 374 374 F F F 14 280-299 182 182 182 372 372 372 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 186 186 186 380 380 380 F F F 17 340-359 186 186 186 381 381 381 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 188 188 188 387 387 387 T T T 20 400-419 186 186 186 381 381 381 F F F 21 420-439 186 186 186 383 383 383 F F F 22 440-459 188 188 188 387 387 387 F F F 23 460-479 186 186 186 380 380 380 F F F 24 480-499 186 186 186 381 381 381 F F F 25 500-519 186 186 186 381 381 381 T T T 26 520-539 188 188 188 387 387 387 T T T 27 540-559 183 183 183 374 374 374 F F F 28 560-579 184 184 184 375 375 375 F F F 29 580-599 184 184 184 375 375 375 F F F Max gen: 188, Max flush: 387 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 188 188 188 387 387 387 F F F 2 040-059 189 189 189 388 388 388 F F F 3 060-079 187 187 187 384 384 384 F F F 4 080-099 182 182 182 371 371 371 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 188 188 188 387 387 387 F F F 8 160-179 186 186 186 383 383 383 F F F 9 180-199 188 188 188 387 387 387 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 186 186 186 381 381 381 F F F 12 240-259 189 189 189 388 388 388 F F F 13 260-279 183 183 183 374 374 374 F F F 14 280-299 182 182 182 372 372 372 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 186 186 186 380 380 380 F F F 17 340-359 186 186 186 381 381 381 F F F 18 360-379 175 175 175 357 357 357 F F F 19 380-399 189 189 189 388 388 388 F F F 20 400-419 186 186 186 381 381 381 F F F 21 420-439 186 186 186 383 383 383 F F F 22 440-459 188 188 188 387 387 387 F F F 23 460-479 186 186 186 380 380 380 F F F 24 480-499 186 186 186 381 381 381 F F F 25 500-519 189 189 189 388 388 388 F F F 26 520-539 189 189 189 388 388 388 F F F 27 540-559 183 183 183 374 374 374 F F F 28 560-579 184 184 184 375 375 375 F F F 29 580-599 184 184 184 375 375 375 F F F Max gen: 189, Max flush: 388 Begin loop 95 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.646923839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.647256174Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.647285254Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942} {"msg":"Crucible d383df7e-ac5a-4fca-8ffe-7638630c8c06 has session id: e115e14b-253f-490a-866b-a6b57a2af8dc","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.647576408Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Upstairs opts: Upstairs UUID: d383df7e-ac5a-4fca-8ffe-7638630c8c06, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.647620748Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Crucible stats registered with UUID: d383df7e-ac5a-4fca-8ffe-7638630c8c06","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.647644239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.647981943Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942} {"msg":"d383df7e-ac5a-4fca-8ffe-7638630c8c06 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648019163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648035384Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648050414Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648064174Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648078304Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648095444Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648109325Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648188015Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648214466Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648231416Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:49020)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.64856694Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:56175)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648605441Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:04:40.648636381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:04:40.648665052Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:04:40.648691312Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648713102Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648727692Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.648741173Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.649218699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.649254299Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.64933482Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.64935585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.649833696Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.649932838Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.650284712Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.650306602Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.650321403Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.650441854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.650460134Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:40.650473604Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.650553495Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:48273)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.650774938Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.651799701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.651834552Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652298918Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652846265Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652869715Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[0]R flush_numbers[0..12]: [384, 387, 388, 384, 371, 371, 387, 387, 383, 387, 387, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652887935Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[0]R generation[0..12]: [187, 188, 189, 187, 182, 182, 188, 188, 186, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652905755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652923686Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[1]R flush_numbers[0..12]: [384, 387, 388, 384, 371, 371, 387, 387, 383, 387, 387, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652941436Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[1]R generation[0..12]: [187, 188, 189, 187, 182, 182, 188, 188, 186, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652958956Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652976846Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[2]R flush_numbers[0..12]: [384, 387, 388, 384, 371, 371, 387, 387, 383, 387, 387, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.652994296Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[2]R generation[0..12]: [187, 188, 189, 187, 182, 182, 188, 188, 186, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653011776Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":Wait for a query_work_queue command to finish before sending IO 1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653033657Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653049997Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653066877Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Next flush: 389","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653083328Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653103828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"":"downstairs","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653121158Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653136538Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"0","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653153538Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"1","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653170409Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"client":"2","":"downstairs","session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653187149Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653202699Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"d383df7e-ac5a-4fca-8ffe-7638630c8c06 is now active with session: e115e14b-253f-490a-866b-a6b57a2af8dc","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653218839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.653234119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942,"session_id":"e115e14b-253f-490a-866b-a6b57a2af8dc"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:04:50.65324945Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 291 len 24576 data: 13 14 16 16 12 13 02/30 Read block 194 len 28672 03/30 Read block 247 len 40960 04/30 Read block 408 len 4096 05/30 Read block 449 len 32768 06/30 Read block 21 len 36864 07/30 Flush 08/30 Write block 28 len 36864 data: 12 14 10 9 7 8 8 9 9 09/30 Write block 464 len 4096 data: 17 10/30 Flush 11/30 Write block 433 len 40960 data: 16 14 14 14 15 15 17 14 19 17 12/30 Read block 386 len 12288 13/30 Read block 228 len 40960 14/30 Read block 24 len 40960 15/30 Write block 356 len 24576 data: 20 16 16 13 12 13 16/30 Flush 17/30 Write block 472 len 24576 data: 11 15 14 18 21 20 18/30 Write block 27 len 12288 data: 12 13 15 19/30 Read block 551 len 8192 20/30 Write block 80 len 16384 data: 11 10 11 13 21/30 Read block 236 len 4096 22/30 Read block 285 len 24576 23/30 Read block 463 len 4096 24/30 Write block 24 len 36864 data: 13 13 13 13 14 16 11 10 8 25/30 Read block 284 len 4096 26/30 Read block 248 len 20480 27/30 Read block 563 len 4096 28/30 Write block 277 len 24576 data: 9 12 11 10 10 12 29/30 Read block 170 len 12288 30/30 Write block 351 len 8192 data: 14 16 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 6 Done Done Done false 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Read 2 Done Done Done false 20 Acked 1019 Write 4 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:04:50.68083648Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 190 190 190 390 390 390 T T T 2 040-059 189 189 189 388 388 388 F F F 3 060-079 187 187 187 384 384 384 F F F 4 080-099 182 182 182 371 371 371 T T T 5 100-119 182 182 182 371 371 371 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 188 188 188 387 387 387 F F F 8 160-179 186 186 186 383 383 383 F F F 9 180-199 188 188 188 387 387 387 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 186 186 186 381 381 381 F F F 12 240-259 189 189 189 388 388 388 F F F 13 260-279 183 183 183 374 374 374 T T T 14 280-299 190 190 190 389 389 389 T T T 15 300-319 178 178 178 364 364 364 F F F 16 320-339 186 186 186 380 380 380 F F F 17 340-359 190 190 190 391 391 391 T T T 18 360-379 190 190 190 391 391 391 F F F 19 380-399 189 189 189 388 388 388 F F F 20 400-419 186 186 186 381 381 381 F F F 21 420-439 190 190 190 391 391 391 F F F 22 440-459 190 190 190 391 391 391 F F F 23 460-479 190 190 190 390 390 390 T T T 24 480-499 186 186 186 381 381 381 F F F 25 500-519 189 189 189 388 388 388 F F F 26 520-539 189 189 189 388 388 388 F F F 27 540-559 183 183 183 374 374 374 F F F 28 560-579 184 184 184 375 375 375 F F F 29 580-599 184 184 184 375 375 375 F F F Max gen: 190, Max flush: 391 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 191 191 191 392 392 392 F F F 2 040-059 189 189 189 388 388 388 F F F 3 060-079 187 187 187 384 384 384 F F F 4 080-099 191 191 191 392 392 392 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 188 188 188 387 387 387 F F F 8 160-179 186 186 186 383 383 383 F F F 9 180-199 188 188 188 387 387 387 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 186 186 186 381 381 381 F F F 12 240-259 189 189 189 388 388 388 F F F 13 260-279 191 191 191 392 392 392 F F F 14 280-299 191 191 191 392 392 392 F F F 15 300-319 178 178 178 364 364 364 F F F 16 320-339 186 186 186 380 380 380 F F F 17 340-359 191 191 191 392 392 392 F F F 18 360-379 190 190 190 391 391 391 F F F 19 380-399 189 189 189 388 388 388 F F F 20 400-419 186 186 186 381 381 381 F F F 21 420-439 190 190 190 391 391 391 F F F 22 440-459 190 190 190 391 391 391 F F F 23 460-479 191 191 191 392 392 392 F F F 24 480-499 186 186 186 381 381 381 F F F 25 500-519 189 189 189 388 388 388 F F F 26 520-539 189 189 189 388 388 388 F F F 27 540-559 183 183 183 374 374 374 F F F 28 560-579 184 184 184 375 375 375 F F F 29 580-599 184 184 184 375 375 375 F F F Max gen: 191, Max flush: 392 Begin loop 96 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.918909571Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.919254745Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.919274595Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950} {"msg":"Crucible 096d0213-324b-4f47-9cf1-393db0d51e74 has session id: 3855d369-6531-425d-a774-8881e27a3c5c","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.91965828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Upstairs opts: Upstairs UUID: 096d0213-324b-4f47-9cf1-393db0d51e74, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.919790772Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Crucible stats registered with UUID: 096d0213-324b-4f47-9cf1-393db0d51e74","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.919817532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920215537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950} {"msg":"096d0213-324b-4f47-9cf1-393db0d51e74 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920251268Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920266428Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920281258Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920299238Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920313479Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920327629Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920351089Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920372769Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.92038766Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920401949Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56678)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920589102Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43317)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920619132Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:04.920892126Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:04.920933336Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:04.920964167Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.920987967Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.921012617Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.921098469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.921410332Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.921437613Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.921452993Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.921467753Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.921944249Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.922061291Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.922526116Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.922552177Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.922566497Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.922579117Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.922592897Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:04.922611728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.922626078Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57877)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.923071554Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.923930474Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.923960585Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.924486401Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925042088Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925069069Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[0]R flush_numbers[0..12]: [384, 392, 388, 384, 392, 371, 387, 387, 383, 387, 387, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925102679Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[0]R generation[0..12]: [187, 191, 189, 187, 191, 182, 188, 188, 186, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.92512156Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.92513992Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[1]R flush_numbers[0..12]: [384, 392, 388, 384, 392, 371, 387, 387, 383, 387, 387, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.92515747Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[1]R generation[0..12]: [187, 191, 189, 187, 191, 182, 188, 188, 186, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.92517504Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.92519283Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[2]R flush_numbers[0..12]: [384, 392, 388, 384, 392, 371, 387, 387, 383, 387, 387, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.92521041Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[2]R generation[0..12]: [187, 191, 189, 187, 191, 182, 188, 188, 186, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925227681Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-07-10T16:05:14.925245471Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925263051Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925285122Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Next flush: 393","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925302072Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925318552Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"":"downstairs","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925335372Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925350542Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"0","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925367652Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"1","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925384583Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"client":"2","":"downstairs","session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925401743Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925417223Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"096d0213-324b-4f47-9cf1-393db0d51e74 is now active with session: 3855d369-6531-425d-a774-8881e27a3c5c","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925433463Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925448344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950,"session_id":"3855d369-6531-425d-a774-8881e27a3c5c"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:14.925469164Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 583 len 20480 data: 9 7 9 12 12 02/30 Read block 189 len 20480 03/30 Write block 578 len 24576 data: 18 15 13 9 9 10 04/30 Read block 108 len 4096 05/30 Write block 574 len 16384 data: 14 13 16 15 06/30 Read block 279 len 8192 07/30 Read block 518 len 20480 08/30 Read block 210 len 40960 09/30 Read block 36 len 4096 10/30 Write block 262 len 28672 data: 13 13 13 13 12 9 5 11/30 Flush 12/30 Write block 323 len 36864 data: 19 18 16 17 14 13 14 13 14 13/30 Read block 417 len 16384 14/30 Read block 538 len 28672 15/30 Write block 381 len 12288 data: 18 18 13 16/30 Read block 353 len 36864 17/30 Write block 296 len 40960 data: 14 12 9 6 9 10 12 13 13 15 18/30 Write block 566 len 8192 data: 11 12 19/30 Write block 261 len 32768 data: 13 14 14 14 14 13 10 6 20/30 Flush 21/30 Write block 441 len 4096 data: 20 22/30 Write block 83 len 16384 data: 14 12 12 11 23/30 Read block 198 len 20480 24/30 Write block 40 len 36864 data: 11 14 14 17 18 13 14 13 14 25/30 Write block 174 len 12288 data: 15 11 11 26/30 Write block 577 len 12288 data: 16 19 16 27/30 Write block 551 len 28672 data: 19 18 16 15 13 10 14 28/30 Write block 33 len 12288 data: 9 9 10 29/30 Read block 181 len 20480 30/30 Read block 221 len 16384 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 9 9 10 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:14.952569288Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 191 191 191 392 392 392 T T T 2 040-059 189 189 189 388 388 388 T T T 3 060-079 187 187 187 384 384 384 F F F 4 080-099 191 191 191 392 392 392 T T T 5 100-119 182 182 182 371 371 371 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 188 188 188 387 387 387 F F F 8 160-179 186 186 186 383 383 383 T T T 9 180-199 188 188 188 387 387 387 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 186 186 186 381 381 381 F F F 12 240-259 189 189 189 388 388 388 F F F 13 260-279 192 192 192 395 395 395 F F F 14 280-299 192 192 192 394 394 394 F F F 15 300-319 192 192 192 394 394 394 F F F 16 320-339 192 192 192 394 394 394 F F F 17 340-359 191 191 191 392 392 392 F F F 18 360-379 190 190 190 391 391 391 F F F 19 380-399 192 192 192 394 394 394 F F F 20 400-419 186 186 186 381 381 381 F F F 21 420-439 190 190 190 391 391 391 F F F 22 440-459 190 190 190 391 391 391 T T T 23 460-479 191 191 191 392 392 392 F F F 24 480-499 186 186 186 381 381 381 F F F 25 500-519 189 189 189 388 388 388 F F F 26 520-539 189 189 189 388 388 388 F F F 27 540-559 183 183 183 374 374 374 T T T 28 560-579 192 192 192 395 395 395 T T T 29 580-599 192 192 192 393 393 393 F F F Max gen: 192, Max flush: 395 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 193 193 193 396 396 396 F F F 2 040-059 193 193 193 396 396 396 F F F 3 060-079 187 187 187 384 384 384 F F F 4 080-099 193 193 193 396 396 396 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 188 188 188 387 387 387 F F F 8 160-179 193 193 193 396 396 396 F F F 9 180-199 188 188 188 387 387 387 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 186 186 186 381 381 381 F F F 12 240-259 189 189 189 388 388 388 F F F 13 260-279 192 192 192 395 395 395 F F F 14 280-299 192 192 192 394 394 394 F F F 15 300-319 192 192 192 394 394 394 F F F 16 320-339 192 192 192 394 394 394 F F F 17 340-359 191 191 191 392 392 392 F F F 18 360-379 190 190 190 391 391 391 F F F 19 380-399 192 192 192 394 394 394 F F F 20 400-419 186 186 186 381 381 381 F F F 21 420-439 190 190 190 391 391 391 F F F 22 440-459 193 193 193 396 396 396 F F F 23 460-479 191 191 191 392 392 392 F F F 24 480-499 186 186 186 381 381 381 F F F 25 500-519 189 189 189 388 388 388 F F F 26 520-539 189 189 189 388 388 388 F F F 27 540-559 193 193 193 396 396 396 F F F 28 560-579 193 193 193 396 396 396 F F F 29 580-599 192 192 192 393 393 393 F F F Max gen: 193, Max flush: 396 Begin loop 97 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.207222249Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.207596414Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.207616924Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958} {"msg":"Crucible 8ffcc2e5-0b2a-4621-8733-015116950f11 has session id: 6351dbf4-0982-4c35-b960-12534761b6a9","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.207932778Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Upstairs opts: Upstairs UUID: 8ffcc2e5-0b2a-4621-8733-015116950f11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.207980279Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Crucible stats registered with UUID: 8ffcc2e5-0b2a-4621-8733-015116950f11","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.2080339Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208284933Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958} {"msg":"8ffcc2e5-0b2a-4621-8733-015116950f11 active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208386254Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208413724Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208429995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208443995Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208457725Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208480155Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208503205Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208517606Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208533066Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208556256Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44633)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208671308Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:29.208711888Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:29.208740859Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:29.208769129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208793239Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.20881842Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48961)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.20884958Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.208918381Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.209460468Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.209497318Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.209526738Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.209553869Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.210029135Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.210142526Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.21046823Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.210491621Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.210505821Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.210643633Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.210661573Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:29.210675113Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.209572819Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:43794)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.209732701Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.210843735Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.210883066Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.211412833Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.21197169Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.21199163Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[0]R flush_numbers[0..12]: [384, 396, 396, 384, 396, 371, 387, 387, 396, 387, 387, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212046221Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[0]R generation[0..12]: [187, 193, 193, 187, 193, 182, 188, 188, 193, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212065661Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212080481Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[1]R flush_numbers[0..12]: [384, 396, 396, 384, 396, 371, 387, 387, 396, 387, 387, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212094951Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[1]R generation[0..12]: [187, 193, 193, 187, 193, 182, 188, 188, 193, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212108962Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212123342Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[2]R flush_numbers[0..12]: [384, 396, 396, 384, 396, 371, 387, 387, 396, 387, 387, 381]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212137512Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[2]R generation[0..12]: [187, 193, 193, 187, 193, 182, 188, 188, 193, 188, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212151642Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212165892Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212186612Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212201383Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Next flush: 397","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212214743Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212228113Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"":"downstairs","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212241523Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212253963Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"0","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212267324Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"1","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212280634Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"client":"2","":"downstairs","session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212294324Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212306644Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"8ffcc2e5-0b2a-4621-8733-015116950f11 is now active with session: 6351dbf4-0982-4c35-b960-12534761b6a9","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212319684Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212331854Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958,"session_id":"6351dbf4-0982-4c35-b960-12534761b6a9"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:39.212344214Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 449 len 16384 data: 15 13 17 18 02/30 Write block 140 len 36864 data: 9 12 12 19 19 17 18 21 19 03/30 Read block 574 len 20480 04/30 Read block 112 len 28672 05/30 Read block 50 len 32768 06/30 Read block 316 len 36864 07/30 Read block 483 len 36864 08/30 Read block 245 len 16384 09/30 Read block 498 len 16384 10/30 Write block 289 len 28672 data: 13 12 14 15 17 17 13 11/30 Flush 12/30 Write block 536 len 12288 data: 16 18 22 13/30 Write block 62 len 8192 data: 11 12 14/30 Write block 52 len 28672 data: 18 14 13 12 16 13 12 15/30 Write block 61 len 8192 data: 15 12 16/30 Read block 99 len 12288 17/30 Read block 539 len 4096 18/30 Flush 19/30 Write block 185 len 28672 data: 6 7 10 12 14 13 18 20/30 Read block 428 len 16384 21/30 Read block 548 len 12288 22/30 Write block 220 len 28672 data: 14 19 19 15 17 15 14 23/30 Write block 328 len 4096 data: 14 24/30 Flush 25/30 Write block 565 len 4096 data: 9 26/30 Write block 338 len 12288 data: 13 16 15 27/30 Write block 326 len 24576 data: 18 15 15 15 14 15 28/30 Write block 366 len 4096 data: 9 29/30 Read block 219 len 24576 30/30 Write block 331 len 24576 data: 16 16 13 16 12 11 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:39.238829851Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 193 193 193 396 396 396 F F F 2 040-059 194 194 194 398 398 398 F F F 3 060-079 194 194 194 398 398 398 F F F 4 080-099 193 193 193 396 396 396 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 194 194 194 397 397 397 F F F 8 160-179 193 193 193 396 396 396 F F F 9 180-199 194 194 194 399 399 399 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 194 194 194 399 399 399 F F F 12 240-259 189 189 189 388 388 388 F F F 13 260-279 192 192 192 395 395 395 F F F 14 280-299 194 194 194 397 397 397 F F F 15 300-319 192 192 192 394 394 394 F F F 16 320-339 194 194 194 399 399 399 T T T 17 340-359 191 191 191 392 392 392 T T T 18 360-379 190 190 190 391 391 391 T T T 19 380-399 192 192 192 394 394 394 F F F 20 400-419 186 186 186 381 381 381 F F F 21 420-439 190 190 190 391 391 391 F F F 22 440-459 194 194 194 397 397 397 F F F 23 460-479 191 191 191 392 392 392 F F F 24 480-499 186 186 186 381 381 381 F F F 25 500-519 189 189 189 388 388 388 F F F 26 520-539 194 194 194 398 398 398 F F F 27 540-559 193 193 193 396 396 396 F F F 28 560-579 193 193 193 396 396 396 T T T 29 580-599 192 192 192 393 393 393 F F F Max gen: 194, Max flush: 399 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 193 193 193 396 396 396 F F F 2 040-059 194 194 194 398 398 398 F F F 3 060-079 194 194 194 398 398 398 F F F 4 080-099 193 193 193 396 396 396 F F F 5 100-119 182 182 182 371 371 371 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 194 194 194 397 397 397 F F F 8 160-179 193 193 193 396 396 396 F F F 9 180-199 194 194 194 399 399 399 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 194 194 194 399 399 399 F F F 12 240-259 189 189 189 388 388 388 F F F 13 260-279 192 192 192 395 395 395 F F F 14 280-299 194 194 194 397 397 397 F F F 15 300-319 192 192 192 394 394 394 F F F 16 320-339 195 195 195 400 400 400 F F F 17 340-359 195 195 195 400 400 400 F F F 18 360-379 195 195 195 400 400 400 F F F 19 380-399 192 192 192 394 394 394 F F F 20 400-419 186 186 186 381 381 381 F F F 21 420-439 190 190 190 391 391 391 F F F 22 440-459 194 194 194 397 397 397 F F F 23 460-479 191 191 191 392 392 392 F F F 24 480-499 186 186 186 381 381 381 F F F 25 500-519 189 189 189 388 388 388 F F F 26 520-539 194 194 194 398 398 398 F F F 27 540-559 193 193 193 396 396 396 F F F 28 560-579 195 195 195 400 400 400 F F F 29 580-599 192 192 192 393 393 393 F F F Max gen: 195, Max flush: 400 Begin loop 98 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.455801624Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.456136228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.456163839Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966} {"msg":"Crucible dd65b622-bc11-4c8d-8e13-590b1f5e8aac has session id: 5279f392-06e0-4f98-8abc-5bd7f90de9e1","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.456629125Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Upstairs opts: Upstairs UUID: dd65b622-bc11-4c8d-8e13-590b1f5e8aac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.456675405Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Crucible stats registered with UUID: dd65b622-bc11-4c8d-8e13-590b1f5e8aac","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.456699835Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.456958119Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966} {"msg":"dd65b622-bc11-4c8d-8e13-590b1f5e8aac active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.4570532Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.4570755Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457091Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457105121Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457119551Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457132901Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457150961Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457167831Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457182161Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457195952Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53439)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457375444Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:53.457575637Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:53.457616087Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53388)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457645227Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:05:53.457752439Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457778249Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.457797379Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.45788379Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.458230155Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.458293056Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.458361556Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.458429997Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.458746112Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.458857253Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.459204157Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.459252248Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.459281028Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.459309528Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.459336879Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:05:53.459359319Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.459533342Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59010)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.460002647Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.46096673Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.46099567Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.461533077Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462123454Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462148125Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[0]R flush_numbers[0..12]: [384, 396, 398, 398, 396, 371, 387, 397, 396, 399, 387, 399]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462166725Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[0]R generation[0..12]: [187, 193, 194, 194, 193, 182, 188, 194, 193, 194, 188, 194]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462184645Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462202435Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[1]R flush_numbers[0..12]: [384, 396, 398, 398, 396, 371, 387, 397, 396, 399, 387, 399]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462220285Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[1]R generation[0..12]: [187, 193, 194, 194, 193, 182, 188, 194, 193, 194, 188, 194]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462237876Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462257326Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[2]R flush_numbers[0..12]: [384, 396, 398, 398, 396, 371, 387, 397, 396, 399, 387, 399]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462286156Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [187, 193, 194, 194, 193, 182, 188, 194, 193, 194, 188, 194]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462318177Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462348907Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462370967Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462394038Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Next flush: 401","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462417828Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462442678Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"":"downstairs","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462459228Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462471469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"0","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462493809Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"1","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462518289Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"client":"2","":"downstairs","session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.46254016Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.4625534Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"dd65b622-bc11-4c8d-8e13-590b1f5e8aac is now active with session: 5279f392-06e0-4f98-8abc-5bd7f90de9e1","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.46257537Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.46259697Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966,"session_id":"5279f392-06e0-4f98-8abc-5bd7f90de9e1"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:03.462619791Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 173 len 36864 data: 14 16 12 12 13 12 12 10 8 03/30 Read block 170 len 24576 04/30 Write block 474 len 4096 data: 15 05/30 Write block 581 len 4096 data: 10 06/30 Write block 256 len 4096 data: 20 07/30 Flush 08/30 Write block 592 len 16384 data: 14 10 10 9 09/30 Write block 51 len 40960 data: 15 19 15 14 13 17 14 13 14 13 10/30 Flush 11/30 Write block 509 len 28672 data: 11 11 12 9 9 9 9 12/30 Write block 405 len 36864 data: 13 12 11 11 9 8 6 7 8 13/30 Write block 129 len 32768 data: 15 22 20 17 17 19 19 17 14/30 Write block 231 len 20480 data: 8 7 9 15 16 15/30 Read block 463 len 12288 16/30 Read block 409 len 12288 17/30 Read block 146 len 16384 18/30 Flush 19/30 Write block 405 len 36864 data: 14 13 12 12 10 9 7 8 9 20/30 Write block 332 len 16384 data: 17 14 17 13 21/30 Write block 110 len 40960 data: 11 10 9 11 13 11 13 11 12 13 22/30 Read block 372 len 8192 23/30 Write block 543 len 4096 data: 13 24/30 Write block 26 len 36864 data: 14 14 15 17 12 11 9 10 10 25/30 Read block 196 len 32768 26/30 Write block 382 len 12288 data: 19 14 11 27/30 Write block 89 len 12288 data: 12 12 12 28/30 Write block 489 len 8192 data: 14 14 29/30 Read block 46 len 20480 30/30 Write block 131 len 28672 data: 21 18 18 20 20 18 20 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:06:03.490229841Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 193 193 193 396 396 396 T T T 2 040-059 196 196 196 403 403 403 F F F 3 060-079 196 196 196 403 403 403 F F F 4 080-099 193 193 193 396 396 396 T T T 5 100-119 182 182 182 371 371 371 T T T 6 120-139 196 196 196 404 404 404 T T T 7 140-159 194 194 194 397 397 397 F F F 8 160-179 196 196 196 402 402 402 F F F 9 180-199 196 196 196 402 402 402 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 196 196 196 404 404 404 F F F 12 240-259 196 196 196 402 402 402 F F F 13 260-279 192 192 192 395 395 395 F F F 14 280-299 194 194 194 397 397 397 F F F 15 300-319 192 192 192 394 394 394 F F F 16 320-339 195 195 195 400 400 400 T T T 17 340-359 195 195 195 400 400 400 F F F 18 360-379 195 195 195 400 400 400 F F F 19 380-399 192 192 192 394 394 394 T T T 20 400-419 196 196 196 404 404 404 T T T 21 420-439 190 190 190 391 391 391 F F F 22 440-459 194 194 194 397 397 397 F F F 23 460-479 196 196 196 402 402 402 F F F 24 480-499 186 186 186 381 381 381 T T T 25 500-519 196 196 196 404 404 404 F F F 26 520-539 194 194 194 398 398 398 F F F 27 540-559 193 193 193 396 396 396 T T T 28 560-579 195 195 195 400 400 400 F F F 29 580-599 196 196 196 403 403 403 F F F Max gen: 196, Max flush: 404 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 384 384 384 F F F 1 020-039 197 197 197 405 405 405 F F F 2 040-059 196 196 196 403 403 403 F F F 3 060-079 196 196 196 403 403 403 F F F 4 080-099 197 197 197 405 405 405 F F F 5 100-119 197 197 197 405 405 405 F F F 6 120-139 197 197 197 405 405 405 F F F 7 140-159 194 194 194 397 397 397 F F F 8 160-179 196 196 196 402 402 402 F F F 9 180-199 196 196 196 402 402 402 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 196 196 196 404 404 404 F F F 12 240-259 196 196 196 402 402 402 F F F 13 260-279 192 192 192 395 395 395 F F F 14 280-299 194 194 194 397 397 397 F F F 15 300-319 192 192 192 394 394 394 F F F 16 320-339 197 197 197 405 405 405 F F F 17 340-359 195 195 195 400 400 400 F F F 18 360-379 195 195 195 400 400 400 F F F 19 380-399 197 197 197 405 405 405 F F F 20 400-419 197 197 197 405 405 405 F F F 21 420-439 190 190 190 391 391 391 F F F 22 440-459 194 194 194 397 397 397 F F F 23 460-479 196 196 196 402 402 402 F F F 24 480-499 197 197 197 405 405 405 F F F 25 500-519 196 196 196 404 404 404 F F F 26 520-539 194 194 194 398 398 398 F F F 27 540-559 197 197 197 405 405 405 F F F 28 560-579 195 195 195 400 400 400 F F F 29 580-599 196 196 196 403 403 403 F F F Max gen: 197, Max flush: 405 Begin loop 99 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.775527451Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b815a43cfc10bcc32090edef97cce2a57967dc5\",\n git_commit_timestamp: \"2024-07-10T15:11:32.000000000Z\",\n git_branch: \"renovate/nbd-0.x\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.775966727Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 8","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.775991777Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974} {"msg":"Crucible 9dac769b-18ae-442e-bf22-c4963c1f60df has session id: 4c5f8626-39bc-4206-b0c7-9e2747dec195","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.776367872Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Upstairs opts: Upstairs UUID: 9dac769b-18ae-442e-bf22-c4963c1f60df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.776414803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Crucible stats registered with UUID: 9dac769b-18ae-442e-bf22-c4963c1f60df","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.776437893Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.776853728Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974} {"msg":"9dac769b-18ae-442e-bf22-c4963c1f60df active request set","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.776877498Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.776892109Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.776906639Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.776920329Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.776942129Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.77696702Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.77699257Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.77701662Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.77704042Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.777063751Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_connection connected from Ok(127.0.0.1:50073)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.777359255Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:06:17.777481936Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_connection connected from Ok(127.0.0.1:44768)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.777499566Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:06:17.777516186Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:06:17.777536117Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.777549007Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.777562387Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.777575807Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.778217105Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.778243596Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.778258166Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.778271666Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.778805873Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.778917144Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.77932818Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.77935521Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.77936966Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.77939572Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.77941031Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:17.779427691Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.77861469Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_connection connected from Ok(127.0.0.1:42951)","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.778814593Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.779813286Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.779846916Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.780401463Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.78099348Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781018311Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[0]R flush_numbers[0..12]: [384, 405, 403, 403, 405, 405, 405, 397, 402, 402, 387, 404]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781036731Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[0]R generation[0..12]: [187, 197, 196, 196, 197, 197, 197, 194, 196, 196, 188, 196]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781054771Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781073131Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[1]R flush_numbers[0..12]: [384, 405, 403, 403, 405, 405, 405, 397, 402, 402, 387, 404]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781090592Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[1]R generation[0..12]: [187, 197, 196, 196, 197, 197, 197, 194, 196, 196, 188, 196]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781108132Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781126112Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[2]R flush_numbers[0..12]: [384, 405, 403, 403, 405, 405, 405, 397, 402, 402, 387, 404]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781148422Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[2]R generation[0..12]: [187, 197, 196, 196, 197, 197, 197, 194, 196, 196, 188, 196]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781170473Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781189243Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781205823Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781222803Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Next flush: 406","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781239344Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781255864Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"":"downstairs","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781279824Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781298634Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"0","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781315885Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"1","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781332825Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"client":"2","":"downstairs","session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781349925Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781365435Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"9dac769b-18ae-442e-bf22-c4963c1f60df is now active with session: 4c5f8626-39bc-4206-b0c7-9e2747dec195","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781382105Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781397315Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974,"session_id":"4c5f8626-39bc-4206-b0c7-9e2747dec195"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-07-10T16:06:27.781412926Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 481 len 28672 data: 14 13 14 16 13 14 17 02/30 Write block 467 len 28672 data: 17 14 12 13 11 12 16 03/30 Read block 574 len 40960 04/30 Write block 400 len 12288 data: 19 15 15 05/30 Read block 542 len 32768 06/30 Write block 575 len 24576 data: 14 17 17 20 17 14 07/30 Write block 327 len 8192 data: 16 16 08/30 Write block 459 len 16384 data: 8 10 12 13 09/30 Write block 591 len 28672 data: 15 15 11 11 10 7 5 10/30 Read block 427 len 8192 11/30 Read block 213 len 16384 12/30 Read block 300 len 24576 13/30 Write block 568 len 16384 data: 11 6 9 9 14/30 Read block 224 len 12288 15/30 Read block 238 len 12288 16/30 Read block 51 len 20480 17/30 Read block 438 len 32768 18/30 Write block 11 len 40960 data: 14 11 13 13 12 15 13 16 14 14 19/30 Read block 379 len 20480 20/30 Write block 570 len 12288 data: 10 10 13 21/30 Read block 433 len 24576 22/30 Read block 205 len 16384 23/30 Read block 191 len 40960 24/30 Write block 144 len 24576 data: 20 18 19 22 20 17 25/30 Write block 254 len 16384 data: 20 22 21 18 26/30 Write block 472 len 12288 data: 13 17 16 27/30 Write block 21 len 20480 data: 14 14 12 14 14 28/30 Read block 365 len 28672 29/30 Write block 95 len 28672 data: 11 11 11 11 13 15 12 30/30 Read block 155 len 40960 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Read 10 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Write 7 Done Done Done false 31 Acked 1030 Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-07-10T16:06:27.807379545Z","hostname":"w-01J2EKZDSC9YSAZ70AS3VCQEFS","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 406 406 406 F F F 1 020-039 198 198 198 406 406 406 T T T 2 040-059 196 196 196 403 403 403 F F F 3 060-079 196 196 196 403 403 403 F F F 4 080-099 197 197 197 405 405 405 T T T 5 100-119 197 197 197 405 405 405 T T T 6 120-139 197 197 197 405 405 405 F F F 7 140-159 194 194 194 397 397 397 T T T 8 160-179 196 196 196 402 402 402 F F F 9 180-199 196 196 196 402 402 402 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 196 196 196 404 404 404 F F F 12 240-259 196 196 196 402 402 402 T T T 13 260-279 192 192 192 395 395 395 F F F 14 280-299 194 194 194 397 397 397 F F F 15 300-319 192 192 192 394 394 394 F F F 16 320-339 198 198 198 406 406 406 F F F 17 340-359 195 195 195 400 400 400 F F F 18 360-379 195 195 195 400 400 400 F F F 19 380-399 197 197 197 405 405 405 F F F 20 400-419 198 198 198 406 406 406 F F F 21 420-439 190 190 190 391 391 391 F F F 22 440-459 198 198 198 406 406 406 F F F 23 460-479 198 198 198 406 406 406 T T T 24 480-499 198 198 198 406 406 406 F F F 25 500-519 196 196 196 404 404 404 F F F 26 520-539 194 194 194 398 398 398 F F F 27 540-559 197 197 197 405 405 405 F F F 28 560-579 198 198 198 406 406 406 F F F 29 580-599 198 198 198 406 406 406 F F F Max gen: 198, Max flush: 406 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 406 406 406 F F F 1 020-039 199 199 199 407 407 407 F F F 2 040-059 196 196 196 403 403 403 F F F 3 060-079 196 196 196 403 403 403 F F F 4 080-099 199 199 199 407 407 407 F F F 5 100-119 199 199 199 407 407 407 F F F 6 120-139 197 197 197 405 405 405 F F F 7 140-159 199 199 199 407 407 407 F F F 8 160-179 196 196 196 402 402 402 F F F 9 180-199 196 196 196 402 402 402 F F F 10 200-219 188 188 188 387 387 387 F F F 11 220-239 196 196 196 404 404 404 F F F 12 240-259 199 199 199 407 407 407 F F F 13 260-279 192 192 192 395 395 395 F F F 14 280-299 194 194 194 397 397 397 F F F 15 300-319 192 192 192 394 394 394 F F F 16 320-339 198 198 198 406 406 406 F F F 17 340-359 195 195 195 400 400 400 F F F 18 360-379 195 195 195 400 400 400 F F F 19 380-399 197 197 197 405 405 405 F F F 20 400-419 198 198 198 406 406 406 F F F 21 420-439 190 190 190 391 391 391 F F F 22 440-459 198 198 198 406 406 406 F F F 23 460-479 199 199 199 407 407 407 F F F 24 480-499 198 198 198 406 406 406 F F F 25 500-519 196 196 196 404 404 404 F F F 26 520-539 194 194 194 398 398 398 F F F 27 540-559 197 197 197 405 405 405 F F F 28 560-579 198 198 198 406 406 406 F F F 29 580-599 198 198 198 406 406 406 F F F Max gen: 199, Max flush: 407 40:09 Test duration Test completed real 40:08.391492927 user 41.323528344 sys 19.235819075 trap 0.053791778 tflt 0.134593268 dflt 0.140405734 kflt 0.005213060 lock 63:17:52.476109052 slp 3:16:48.614029538 lat 8.586071879 stop 0.065151546